From 70619e827b3fea536a0390fcc01befbda852c3c6 Mon Sep 17 00:00:00 2001
From: Philippe Canal <pcanal@fnal.gov>
Date: Sat, 21 Jun 2008 03:13:04 +0000
Subject: [PATCH] Disk and Memory Space Gain
In ROOT older than v5.20/00, the branches' last basket, also known as the write basket, was always saved in the same "key" as the TTree object and was always present in memory when reading or writing. When reading this write basket was always present in memory even if the branch was never accessed.
Starting in v5.20/00, TTree::Write closes out, compresses (when requested) and writes to disk in their own file record the write baskets of all the branches. (This is implemented via the new function TTree::FlushBaskets, TBranch::FlushBaskets, TBranch::FlushOneBaskets)
TTree::AutoSave supports a new option "FlushBaskets" which will call FlushBaskets before saving the TTree object.
Benefits
- Flushing the write baskets has several advantages:
- Reduce the file size of the TTree object (it not longer contains the last basket), improving read time of the TTree object
- Reduce memory footprint of the TTree object.
- In a TTree which "flushed" buffer, there is now usually only zero or one buffer in memory.
- Previously each branch always had at least one basket in memory and usually 2 (the write basket and one read basket).
- Now only the basket of the branches actually read are loaded in memory.
- allow for the basket to be compressed and stored separated, increasing the compression factor.
Note: Calling FlushBaskets too often (either directly of via AutoSave("FlushBaskets")) can lead to unnecessary fragmentation of the ROOT file, since it write the baskets to disk (and a new basket will be started at the next fill) whether or not the content was close to filling the basket or not.
The fast tree cloning (TTreeCloner) was enhanced to support copying in-memory TTrees (that have been save as a single key on file). This issue was preventing hadd to fast clone files containing any 'in-memory' tree.
git-svn-id: http://root.cern.ch/svn/root/trunk@24454 27541ba8-7e3a-0410-8455-c3a389f83636
---
tree/doc/v520/index.html | 33 +++++-
tree/tree/inc/TBranch.h | 7 +-
tree/tree/inc/TTree.h | 3 +
tree/tree/src/TBasket.cxx | 4 +-
tree/tree/src/TBranch.cxx | 186 +++++++++++++++++++++----------
tree/tree/src/TBranchClones.cxx | 3 -
tree/tree/src/TBranchElement.cxx | 15 +--
tree/tree/src/TBranchObject.cxx | 5 -
tree/tree/src/TBranchRef.cxx | 3 -
tree/tree/src/TBranchSTL.cxx | 10 --
tree/tree/src/TTree.cxx | 54 +++++++++
tree/tree/src/TTreeCloner.cxx | 63 ++++-------
12 files changed, 249 insertions(+), 137 deletions(-)
diff --git a/tree/doc/v520/index.html b/tree/doc/v520/index.html
index ecd6fc80957..7aedd4d0b3b 100644
--- a/tree/doc/v520/index.html
+++ b/tree/doc/v520/index.html
@@ -129,6 +129,7 @@ where reco_ee_et is a vector<vector<double> > See http://root.cern.
<li>Insure that the formula that are used as indices or as argument to special functions have their branch(es) loaded once. This fixes http://root.cern.ch/phpBB2/viewtopic.php?p=27080#27080
<li>Correct the drawing of "X[1]:X[5]" when X is a vector< vector<float> >
and X[1].size()!=X[5].size(). (reported at http://root.cern.ch/phpBB2/viewtopic.php?p=27070)
+<li>Correct the passing of NaN to function being called by TTree::Draw.</li>
</ul>
<h4>Splitting STL collections of pointers</h4>
@@ -156,8 +157,37 @@ The size of this unzipping cache is 20% the size of the TTreeCache and can be mo
This experimental feature is disabled by default, to activate it use the static function <pre>TTreeCache::SetParallelUnzip(TTreeCacheUnzip::EParUnzipMode option = TTreeCacheUnzip::kEnable).</pre> The possible values to pass are: <ul><li>TTreeCacheUnzip::kEnable to enable it</li><li>TTreeCacheUnzip::kDisable to disable it</li><li>TTreeCacheUnzip::kForce to force it.</li></ul>The TTreeCacheUnzip is actived
only if you have more than one core. To activate it with only one core useTTreeCacheUnzip::kForce option (for example to measure the overhead).
+<h4>Disk and Memory Space Gain</h4>
+
+In ROOT older than v5.20/00, the branches' last basket, also known as the <i>write basket</i>, was always saved in the same "key" as the TTree object and was always present in memory when reading or writing.
+When reading this write basket was always present in memory even if the branch was never accessed.
+<p>
+Starting in v5.20/00, TTree::Write closes out, compresses (when requested) and writes to disk in their own file record the <i>write baskets</i> of all the branches.
+(This is implemented via the new function TTree::FlushBaskets, TBranch::FlushBaskets, TBranch::FlushOneBaskets)
+<p>
+TTree::AutoSave supports a new option "FlushBaskets" which will call FlushBaskets before saving the TTree object.
+
+<h5><b><u>Benefits</u></b></h5>
+
+Flushing the write baskets has several advantages:
+<ul>
+<li>Reduce the file size of the TTree object (it not longer contains the last basket), improving read time of the TTree object</li>
+<li>Reduce memory footprint of the TTree object.
+<ul><li>In a TTree which "flushed" buffer, there is now usually only zero or one buffer in memory.</li>
+<li>Previously each branch always had at least one basket in memory and usually 2 (the write basket and one read basket).</li>
+<li>Now only the basket of the branches actually read are loaded in memory.</li>
+</ul>
+</li>
+<li>allow for the basket to be compressed and stored separated, increasing the compression factor.</li>
+</ul>
+
+Note: Calling FlushBaskets too often (either directly of via AutoSave("FlushBaskets")) can lead to unnecessary fragmentation of the ROOT file,
+since it write the baskets to disk (and a new basket will be started at the next fill) whether or not the content was close to filling the basket or not.
+
<h4>Others</h4>
<ul>
+<li>The fast tree cloning (TTreeCloner) was enhanced to support copying in-memory TTrees (that have been save as a single key on file). This issue was preventing <b>hadd</b> to fast clone files containing any 'in-memory' tree.
+</li>
<li>Re-enabled the splitting of TVector3 and of any classes starting by TVector
that is not a TVectorT.</li>
<li>Fix the list of StreamerInfo stored in the TFile in the case of a slow
@@ -193,5 +223,6 @@ This omission meant that slow CloneTree was (fataly) missing in
some cases the copy of the TStreamerInfo for class that are part
part of the TTree but had only a base and no member or in
some cases where it had only object data members.</li>
+<li>Fix the return value of the lookup in TChainIndex
+when the value searched for does not exist.</li>
</ul>
-
diff --git a/tree/tree/inc/TBranch.h b/tree/tree/inc/TBranch.h
index 5159411d51d..b6571da5f3b 100644
--- a/tree/tree/inc/TBranch.h
+++ b/tree/tree/inc/TBranch.h
@@ -93,6 +93,8 @@ protected:
void SetSkipZip(Bool_t skip = kTRUE) { fSkipZip = skip; }
void Init(const char *name, const char *leaflist, Int_t compress);
+ Int_t WriteBasket(TBasket* basket, Int_t where);
+
private:
TBranch(const TBranch&); // not implemented
TBranch& operator=(const TBranch&); // not implemented
@@ -104,6 +106,7 @@ public:
virtual ~TBranch();
virtual void AddBasket(TBasket &b, Bool_t ondisk, Long64_t startEntry);
+ virtual void AddLastBasket(Long64_t startEntry);
virtual void Browse(TBrowser *b);
virtual void DeleteBaskets(Option_t* option="");
virtual void DropBaskets(Option_t *option = "");
@@ -112,6 +115,9 @@ public:
virtual void FillLeaves(TBuffer &b);
virtual TBranch *FindBranch(const char *name);
virtual TLeaf *FindLeaf(const char *name);
+ Int_t FlushBaskets();
+ Int_t FlushOneBasket(UInt_t which);
+
virtual char *GetAddress() const {return fAddress;}
virtual Int_t GetBasketSize() const {return fBasketSize;}
virtual const char* GetClassName() const { return ""; }
@@ -175,7 +181,6 @@ public:
virtual void SetOffset(Int_t offset=0) {fOffset=offset;}
virtual void SetTree(TTree *tree) { fTree = tree;}
virtual void UpdateAddress() {;}
- void WriteBasket(TBasket* basket);
static void ResetCount();
diff --git a/tree/tree/inc/TTree.h b/tree/tree/inc/TTree.h
index 66b42ebecaa..17ee9b373e0 100644
--- a/tree/tree/inc/TTree.h
+++ b/tree/tree/inc/TTree.h
@@ -247,6 +247,7 @@ public:
virtual TBranch *FindBranch(const char* name);
virtual TLeaf *FindLeaf(const char* name);
virtual Int_t Fit(const char* funcname, const char* varexp, const char* selection = "", Option_t* option = "", Option_t* goption = "", Long64_t nentries = 1000000000, Long64_t firstentry = 0); // *MENU*
+ virtual Int_t FlushBaskets() const;
virtual const char *GetAlias(const char* aliasName) const;
virtual TBranch *GetBranch(const char* name);
virtual TBranchRef *GetBranchRef() const { return fBranchRef; };
@@ -400,6 +401,8 @@ public:
virtual void StartViewer(); // *MENU*
virtual Int_t UnbinnedFit(const char* funcname, const char* varexp, const char* selection = "", Option_t* option = "", Long64_t nentries = 1000000000, Long64_t firstentry = 0);
void UseCurrentStyle();
+ virtual Int_t Write(const char *name=0, Int_t option=0, Int_t bufsize=0);
+ virtual Int_t Write(const char *name=0, Int_t option=0, Int_t bufsize=0) const;
ClassDef(TTree,16) //Tree descriptor (the main ROOT I/O class)
};
diff --git a/tree/tree/src/TBasket.cxx b/tree/tree/src/TBasket.cxx
index fb013adb12f..541019f09c3 100644
--- a/tree/tree/src/TBasket.cxx
+++ b/tree/tree/src/TBasket.cxx
@@ -53,7 +53,7 @@ TBasket::TBasket()
//_______________________________________________________________________
TBasket::TBasket(TDirectory *motherDir) : TKey(motherDir)
{
- // Simple Constructor.
+ // Constructor used during reading.
fDisplacement = 0;
fEntryOffset = 0;
@@ -71,7 +71,7 @@ TBasket::TBasket(TDirectory *motherDir) : TKey(motherDir)
TBasket::TBasket(const char *name, const char *title, TBranch *branch) :
TKey(branch->GetDirectory())
{
- // Basket normal constructor.
+ // Basket normal constructor, used during writing.
SetName(name);
SetTitle(title);
diff --git a/tree/tree/src/TBranch.cxx b/tree/tree/src/TBranch.cxx
index ddef833c89c..489aab5319a 100644
--- a/tree/tree/src/TBranch.cxx
+++ b/tree/tree/src/TBranch.cxx
@@ -85,7 +85,7 @@ TBranch::TBranch()
, fZipBytes(0)
, fBranches()
, fLeaves()
-, fBaskets()
+, fBaskets(fMaxBaskets)
, fNBasketRAM(kMaxRAM+1)
, fBasketRAM(0)
, fBasketBytes(0)
@@ -129,7 +129,7 @@ TBranch::TBranch(TTree *tree, const char* name, void* address, const char* leafl
, fZipBytes(0)
, fBranches()
, fLeaves()
-, fBaskets()
+, fBaskets(fMaxBaskets)
, fNBasketRAM(kMaxRAM+1)
, fBasketRAM(0)
, fBasketBytes(0)
@@ -211,7 +211,7 @@ TBranch::TBranch(TBranch *parent, const char* name, void* address, const char* l
, fZipBytes(0)
, fBranches()
, fLeaves()
-, fBaskets()
+, fBaskets(fMaxBaskets)
, fNBasketRAM(kMaxRAM+1)
, fBasketRAM(0)
, fBasketBytes(0)
@@ -403,15 +403,6 @@ void TBranch::Init(const char* name, const char* leaflist, Int_t compress)
delete[] leaftype;
leaftype = 0;
- // Create the first basket.
- //
- // Note: We cannot do this until we have created the leaves,
- // otherwise we do not know the fEntryOffsetLen which
- // compensates for variable-sized leaves, like varying
- // length arrays.
- //
- TBasket* basket = fTree->CreateBasket(this);
- fBaskets.AddAt(basket, 0);
}
//______________________________________________________________________________
@@ -530,12 +521,10 @@ void TBranch::AddBasket(TBasket& b, Bool_t ondisk, Long64_t startEntry)
if (ondisk) {
fBasketBytes[where] = basket->GetNbytes(); // not for in mem
fBasketSeek[where] = basket->GetSeekKey(); // not for in mem
+ fBaskets.AddAtAndExpand(0,fWriteBasket);
++fWriteBasket;
} else {
fBaskets.AddAtAndExpand(basket,fWriteBasket);
- if (fWriteBasket >= fMaxBaskets) {
- ExpandBasketArrays();
- }
fTree->IncrementTotalBuffers(basket->GetBufferSize());
}
@@ -549,6 +538,26 @@ void TBranch::AddBasket(TBasket& b, Bool_t ondisk, Long64_t startEntry)
}
}
+//______________________________________________________________________________
+void TBranch::AddLastBasket(Long64_t startEntry)
+{
+ // Add the start entry of the write basket (not yet create)
+
+ if (fWriteBasket >= fMaxBaskets) {
+ ExpandBasketArrays();
+ }
+ Int_t where = fWriteBasket;
+
+ if (where && startEntry < fBasketEntry[where-1]) {
+ // Need to find the right location and move the possible baskets
+
+ Fatal("AddBasket","The last basket must have the highest entry number (%s/%d/%d).",GetName(),startEntry,fWriteBasket);
+
+ }
+ fBasketEntry[where] = startEntry;
+ fBaskets.AddAtAndExpand(0,fWriteBasket);
+}
+
//______________________________________________________________________________
void TBranch::Browse(TBrowser* b)
{
@@ -688,8 +697,11 @@ void TBranch::ExpandBasketArrays()
newsize*sizeof(Long64_t),fMaxBaskets*sizeof(Long64_t));
fBasketSeek = (Long64_t*)TStorage::ReAlloc(fBasketSeek,
newsize*sizeof(Long64_t),fMaxBaskets*sizeof(Long64_t));
+
fMaxBaskets = newsize;
+ fBaskets.Expand(newsize);
+
for (Int_t i=fWriteBasket;i<fMaxBaskets;i++) {
fBasketBytes[i] = 0;
fBasketEntry[i] = 0;
@@ -714,7 +726,9 @@ Int_t TBranch::Fill()
TBasket* basket = GetBasket(fWriteBasket);
if (!basket) {
- return 0;
+ basket = fTree->CreateBasket(this); // create a new basket
+ if (!basket) return 0;
+ fBaskets.AddAtAndExpand(basket,fWriteBasket);
}
TBuffer* buf = basket->GetBufferRef();
@@ -744,7 +758,7 @@ Int_t TBranch::Fill()
// If the basket already contains entry we need to close it
// out. (This is because we can only transfer full compressed
// buffer)
- WriteBasket(basket);
+ WriteBasket(basket,fWriteBasket);
// And restart from scratch
return Fill();
}
@@ -836,26 +850,7 @@ Int_t TBranch::Fill()
if (fTree->TestBit(TTree::kCircular)) {
return nbytes;
}
- Int_t nout = basket->WriteBuffer();
- fBasketBytes[fWriteBasket] = basket->GetNbytes();
- fBasketSeek[fWriteBasket] = basket->GetSeekKey();
- Int_t addbytes = basket->GetObjlen() + basket->GetKeylen();
- if (fDirectory && (fDirectory != gROOT) && fDirectory->IsWritable()) {
- delete basket;
- basket = 0;
- fBaskets[fWriteBasket] = 0;
- }
- fZipBytes += nout;
- fTotBytes += addbytes;
- fTree->AddTotBytes(addbytes);
- fTree->AddZipBytes(nout);
- basket = fTree->CreateBasket(this);
- ++fWriteBasket;
- fBaskets.AddAtAndExpand(basket, fWriteBasket);
- if (fWriteBasket >= fMaxBaskets) {
- ExpandBasketArrays();
- }
- fBasketEntry[fWriteBasket] = fEntryNumber;
+ Int_t nout = WriteBasket(basket,fWriteBasket);
return (nout >= 0) ? nbytes : -1;
}
return nbytes;
@@ -959,6 +954,79 @@ TLeaf* TBranch::FindLeaf(const char* searchname)
return 0;
}
+
+//______________________________________________________________________________
+Int_t TBranch::FlushBaskets()
+{
+ // Flush to disk all the baskets of this branch and any of subbranches.
+ // Return the number of bytes written or -1 in case of write error.
+
+ UInt_t nerror = 0;
+ Int_t nbytes = 0;
+
+ for(Int_t i=0; i<=fWriteBasket; ++i) {
+ if (fBaskets.UncheckedAt(i)) {
+ Int_t nwrite = FlushOneBasket(i);
+ if (nwrite<0) {
+ ++nerror;
+ } else {
+ nbytes += nwrite;
+ }
+ }
+ }
+ Int_t len = fBranches.GetEntriesFast();
+ for (Int_t i = 0; i < len; ++i) {
+ TBranch* branch = (TBranch*) fBranches.UncheckedAt(i);
+ if (!branch) {
+ continue;
+ }
+ Int_t nwrite = branch->FlushBaskets();
+ if (nwrite<0) {
+ ++nerror;
+ } else {
+ nbytes += nwrite;
+ }
+ }
+ if (nerror) {
+ return -1;
+ } else {
+ return nbytes;
+ }
+}
+
+//______________________________________________________________________________
+Int_t TBranch::FlushOneBasket(UInt_t ibasket)
+{
+ // If we have a write basket in memory and it contains some entries and
+ // has not yet been written to disk, we write it and delete it from memory.
+ // Return the number of bytes written;
+
+ Int_t nbytes = 0;
+ if (fBaskets.GetEntries()) {
+ TBasket *basket = (TBasket*)fBaskets.UncheckedAt(ibasket);
+
+ if (basket) {
+ if (basket->GetNevBuf()
+ && fBasketSeek[ibasket]==0) {
+ // If the basket already contains entry we need to close it out.
+ // (This is because we can only transfer full compressed buffer)
+
+ if (basket->GetBufferRef()->IsReading()) {
+ basket->SetWriteMode();
+ }
+ nbytes = WriteBasket(basket,ibasket); // WriteBasket will delete the existing basket.
+
+ } else {
+ // If the basket is empty or has already been written.
+ basket->DropBuffers();
+ delete basket;
+ fBaskets[ibasket] = 0;
+ }
+ }
+ }
+ return nbytes;
+}
+
//______________________________________________________________________________
TBasket* TBranch::GetBasket(Int_t basketnumber)
{
@@ -971,8 +1039,9 @@ TBasket* TBranch::GetBasket(Int_t basketnumber)
if (basketnumber <0 || basketnumber > fWriteBasket) return 0;
TBasket *basket = (TBasket*)fBaskets.UncheckedAt(basketnumber);
if (basket) return basket;
+ if (basketnumber == fWriteBasket) return 0;
- // create/decode basket parameters from buffer
+ // create/decode basket parameters from buffer
TFile *file = GetFile(0);
basket = new TBasket(file);
if (fSkipZip) basket->SetBit(TBufferFile::kNotDecompressed);
@@ -1574,7 +1643,7 @@ void TBranch::Refresh(TBranch* b)
Int_t nbaskets = b->fBaskets.GetSize();
fBaskets.Expand(nbaskets);
//The current fWritebasket must always be in memory.
- //Take it (just s swap) from the Tree being read
+ //Take it (just swap) from the Tree being read
TBasket *basket = (TBasket*)b->fBaskets.UncheckedAt(fWriteBasket);
fBaskets.AddAt(basket,fWriteBasket);
b->fBaskets.RemoveAt(fWriteBasket);
@@ -1598,8 +1667,6 @@ void TBranch::Reset(Option_t*)
fZipBytes = 0;
fEntryNumber = 0;
- Int_t nbaskets = fBaskets.GetEntries();
-
if (fBasketBytes) {
for (Int_t i = 0; i < fMaxBaskets; ++i) {
fBasketBytes[i] = 0;
@@ -1619,11 +1686,6 @@ void TBranch::Reset(Option_t*)
}
fBaskets.Delete();
-
- if (nbaskets) {
- TBasket* basket = fTree->CreateBasket(this);
- fBaskets.AddAt(basket, 0);
- }
}
//______________________________________________________________________________
@@ -1854,6 +1916,7 @@ void TBranch::Streamer(TBuffer& b)
TLeaf *leaf = (TLeaf*)fLeaves.UncheckedAt(i);
leaf->SetBranch(this);
}
+
Int_t nbaskets = fBaskets.GetEntries();
for (Int_t j=fWriteBasket,n=0;j>=0 && n<nbaskets;--j) {
TBasket *bk = (TBasket*)fBaskets.UncheckedAt(j);
@@ -1991,7 +2054,7 @@ void TBranch::Streamer(TBuffer& b)
} else {
Int_t maxBaskets = fMaxBaskets;
- fMaxBaskets = fBaskets.GetEntriesFast();
+ fMaxBaskets = fWriteBasket+1;
if (fMaxBaskets < 10) fMaxBaskets=10;
b.WriteClassBuffer(TBranch::Class(),this);
fMaxBaskets = maxBaskets;
@@ -1999,30 +2062,35 @@ void TBranch::Streamer(TBuffer& b)
}
//_______________________________________________________________________
-void TBranch::WriteBasket(TBasket* basket)
+Int_t TBranch::WriteBasket(TBasket* basket, Int_t where)
{
- // Write the current basket to disk
+ // Write the current basket to disk and return the number of bytes
+ // written to the file.
Int_t nout = basket->WriteBuffer(); // Write buffer
- fBasketBytes[fWriteBasket] = basket->GetNbytes();
- fBasketSeek[fWriteBasket] = basket->GetSeekKey();
+ fBasketBytes[where] = basket->GetNbytes();
+ fBasketSeek[where] = basket->GetSeekKey();
Int_t addbytes = basket->GetObjlen() + basket->GetKeylen() ;
- if (fDirectory != gROOT && fDirectory->IsWritable()) {
+ if (fDirectory && (fDirectory != gROOT) && fDirectory->IsWritable()) {
+ basket->DropBuffers();
delete basket;
- fBaskets[fWriteBasket] = 0;
+ fBaskets[where] = 0;
}
fZipBytes += nout;
fTotBytes += addbytes;
fTree->AddTotBytes(addbytes);
fTree->AddZipBytes(nout);
- basket = fTree->CreateBasket(this); // create a new basket
- fWriteBasket++;
- fBaskets.AddAtAndExpand(basket,fWriteBasket);
- if (fWriteBasket >= fMaxBaskets) {
- ExpandBasketArrays();
+ if (where==fWriteBasket) {
+ ++fWriteBasket;
+ if (fWriteBasket >= fMaxBaskets) {
+ ExpandBasketArrays();
+ }
+ fBaskets.AddAtAndExpand(0,fWriteBasket);
+ fBasketEntry[fWriteBasket] = fEntryNumber;
}
- fBasketEntry[fWriteBasket] = fEntryNumber;
+
+ return nout;
}
//------------------------------------------------------------------------------
diff --git a/tree/tree/src/TBranchClones.cxx b/tree/tree/src/TBranchClones.cxx
index 5c28f666649..9b1f187dabb 100644
--- a/tree/tree/src/TBranchClones.cxx
+++ b/tree/tree/src/TBranchClones.cxx
@@ -123,9 +123,6 @@ void TBranchClones::Init(TTree *tree, TBranch *parent, const char* name, void* p
fDirectory = fTree->GetDirectory();
fFileName = "";
- // Create the first basket.
- TBasket* basket = new TBasket(branchcount, fTree->GetName(), this);
- fBaskets.Add(basket);
// Loop on all public data members of the class and its base classes.
const char* itype = 0;
TRealData* rd = 0;
diff --git a/tree/tree/src/TBranchElement.cxx b/tree/tree/src/TBranchElement.cxx
index a8520be26f5..57c32764412 100644
--- a/tree/tree/src/TBranchElement.cxx
+++ b/tree/tree/src/TBranchElement.cxx
@@ -262,10 +262,6 @@ void TBranchElement::Init(TTree *tree, TBranch *parent,const char* bname, TStrea
fBasketSeek[i] = 0;
}
- // Create a basket for the branch.
- TBasket* basket = new TBasket(name, fTree->GetName(), this);
- fBaskets.Add(basket);
-
// We need to keep track of the counter branch if we have
// one, since we cannot set it until we have created our
// leaf, which we do last.
@@ -393,7 +389,7 @@ void TBranchElement::Init(TTree *tree, TBranch *parent,const char* bname, TStrea
} else {
clones = (TClonesArray*)pointer;
}
- basket->DeleteEntryOffset(); //entryoffset not required for the clonesarray counter
+// basket->DeleteEntryOffset(); //entryoffset not required for the clonesarray counter
fEntryOffsetLen = 0;
// ===> Create a leafcount
TLeaf* leaf = new TLeafElement(this, name, fID, fStreamerType);
@@ -603,10 +599,6 @@ void TBranchElement::Init(TTree *tree, TBranch *parent, const char* bname, TClon
fBasketSeek[i] = 0;
}
- // Create a basket for the terminal branch
- TBasket *basket = new TBasket(name, fTree->GetName(), this);
- fBaskets.Add(basket);
-
// Reset the bit kAutoDelete to specify that when reading
// the object should not be deleted before calling the streamer.
SetAutoDelete(kFALSE);
@@ -734,10 +726,7 @@ void TBranchElement::Init(TTree *tree, TBranch *parent, const char* bname, TVirt
fBasketEntry[0] = fEntryNumber;
fBasketBytes[0] = 0;
-
- // Create a basket for the terminal branch
- TBasket* basket = new TBasket(name, fTree->GetName(), this);
- fBaskets.Add(basket);
+ fBasketSeek[0] = 0;
// Reset the bit kAutoDelete to specify that, when reading,
// the object should not be deleted before calling the streamer.
diff --git a/tree/tree/src/TBranchObject.cxx b/tree/tree/src/TBranchObject.cxx
index 836114772e4..ee90082848b 100644
--- a/tree/tree/src/TBranchObject.cxx
+++ b/tree/tree/src/TBranchObject.cxx
@@ -142,11 +142,6 @@ void TBranchObject::Init(TTree *tree, TBranch *parent, const char* name, const c
fDirectory = fTree->GetDirectory();
fFileName = "";
- // Create the first basket.
- if (!splitlevel) {
- TBasket* basket = new TBasket(name, fTree->GetName(), this);
- fBaskets.Add(basket);
- }
}
//______________________________________________________________________________
diff --git a/tree/tree/src/TBranchRef.cxx b/tree/tree/src/TBranchRef.cxx
index 5805f34a379..da92f6d9a0a 100644
--- a/tree/tree/src/TBranchRef.cxx
+++ b/tree/tree/src/TBranchRef.cxx
@@ -78,9 +78,6 @@ TBranchRef::TBranchRef(TTree *tree)
fDirectory = fTree->GetDirectory();
fFileName = "";
- // Create the first basket
- TBasket *basket = new TBasket("TRefTable",fTree->GetName(),this);
- fBaskets.Add(basket);
}
diff --git a/tree/tree/src/TBranchSTL.cxx b/tree/tree/src/TBranchSTL.cxx
index 92d1d1bbd09..12405267b7b 100644
--- a/tree/tree/src/TBranchSTL.cxx
+++ b/tree/tree/src/TBranchSTL.cxx
@@ -75,11 +75,6 @@ TBranchSTL::TBranchSTL( TTree *tree, const char *name,
fBasketSeek[i] = 0;
}
- //---------------------------------------------------------------------------
- // Create a basket for the branch.
- //---------------------------------------------------------------------------
- TBasket* basket = new TBasket(name, fTree->GetName(), this);
- fBaskets.Add(basket);
}
//------------------------------------------------------------------------------
@@ -123,11 +118,6 @@ TBranchSTL::TBranchSTL( TBranch* parent, const char* name,
fBasketSeek[i] = 0;
}
- //---------------------------------------------------------------------------
- // Create a basket for the branch.
- //---------------------------------------------------------------------------
- TBasket* basket = new TBasket(name, fTree->GetName(), this);
- fBaskets.Add(basket);
}
//------------------------------------------------------------------------------
diff --git a/tree/tree/src/TTree.cxx b/tree/tree/src/TTree.cxx
index 4aef1c5bdce..089e394e183 100644
--- a/tree/tree/src/TTree.cxx
+++ b/tree/tree/src/TTree.cxx
@@ -899,6 +899,9 @@ Long64_t TTree::AutoSave(Option_t* option)
// if option contains "SaveSelf", gDirectory->SaveSelf() is called.
// This allows another process to analyze the Tree while the Tree is being filled.
//
+ // if option contains "FlushBaskets", TTree::FlushBaskets is called and all
+ // the current basket are closed-out and written to disk individually.
+ //
// By default the previous header is deleted after having written the new header.
// if option contains "Overwrite", the previous Tree header is deleted
// before written the new header. This option is slightly faster, but
@@ -950,6 +953,9 @@ Long64_t TTree::AutoSave(Option_t* option)
}
TString opt = option;
opt.ToLower();
+
+ if (opt.Contains("flushbaskets")) FlushBaskets();
+
fSavedBytes = fTotBytes;
TKey *key = (TKey*)fDirectory->GetListOfKeys()->FindObject(GetName());
@@ -3651,6 +3657,35 @@ Int_t TTree::Fit(const char* funcname, const char* varexp, const char* selection
return -1;
}
+//______________________________________________________________________________
+Int_t TTree::FlushBaskets() const
+{
+ // Write to disk all the basket that have not yet been individually written.
+ //
+ // Return the number of bytes written or -1 in case of write error.
+
+ Int_t nbytes = 0;
+ Int_t nerror = 0;
+ TObjArray *lb = const_cast<TTree*>(this)->GetListOfBranches();
+ Int_t nb = lb->GetEntriesFast();
+ for (Int_t j = 0; j < nb; j++) {
+ TBranch* branch = (TBranch*) lb->UncheckedAt(j);
+ if (branch) {
+ Int_t nwrite = branch->FlushBaskets();
+ if (nwrite<0) {
+ ++nerror;
+ } else {
+ nbytes += nwrite;
+ }
+ }
+ }
+ if (nerror) {
+ return -1;
+ } else {
+ return nbytes;
+ }
+}
+
//______________________________________________________________________________
const char* TTree::GetAlias(const char* aliasName) const
{
@@ -6327,6 +6362,25 @@ void TTree::UseCurrentStyle()
}
}
+//______________________________________________________________________________
+Int_t TTree::Write(const char *name, Int_t option, Int_t bufsize) const
+{
+ // Write this object to the current directory. For more see TObject::Write
+ // Write calls TTree::FlushBaskets before writing the tree.
+
+ FlushBaskets();
+ return TObject::Write(name, option, bufsize);
+}
+
+//______________________________________________________________________________
+Int_t TTree::Write(const char *name, Int_t option, Int_t bufsize)
+{
+ // Write this object to the current directory. For more see TObject::Write
+ // If option & kFlushBasket, call FlushBasket before writing the tree.
+
+ return ((const TTree*)this)->Write(name, option, bufsize);
+}
+
//////////////////////////////////////////////////////////////////////////
// //
// TTreeFriendLeafIter //
diff --git a/tree/tree/src/TTreeCloner.cxx b/tree/tree/src/TTreeCloner.cxx
index 66de7373755..8de44e0cf49 100644
--- a/tree/tree/src/TTreeCloner.cxx
+++ b/tree/tree/src/TTreeCloner.cxx
@@ -136,29 +136,7 @@ void TTreeCloner::CloseOutWriteBaskets()
for(Int_t i=0; i<fToBranches.GetEntries(); ++i) {
TBranch *to = (TBranch*)fToBranches.UncheckedAt(i);
-
- TObjArray *array = to->GetListOfBaskets();
- if (array->GetEntries()) {
- TBasket *basket = to->GetBasket(to->GetWriteBasket());
- if (basket) {
- if (basket->GetNevBuf()) {
- // If the basket already contains entry we need to close it
- // out. (This is because we can only transfer full compressed
- // buffer)
-
- if (basket->GetBufferRef()->IsReading()) {
- basket->SetWriteMode();
- }
- to->WriteBasket(basket);
- basket = to->GetBasket(to->GetWriteBasket()); // WriteBasket create an empty basket
- }
- if (basket) {
- basket->DropBuffers();
- to->GetListOfBaskets()->RemoveAt(to->GetWriteBasket());
- delete basket;
- }
- }
- }
+ to->FlushOneBasket(to->GetWriteBasket());
}
}
@@ -352,10 +330,12 @@ void TTreeCloner::CopyMemoryBaskets()
basket = (TBasket*)basket->Clone();
basket->SetBranch(to);
to->AddBasket(*basket, kFALSE, fToStartEntries+from->GetBasketEntry()[from->GetWriteBasket()]);
+ } else {
+ to->AddLastBasket( fToStartEntries+from->GetBasketEntry()[from->GetWriteBasket()] );
}
- // If the branch is a TBranchElement non-terminal 'object' branch, it's basket will contain 0
- // events.
- if (from->GetEntries()!=0 && from->GetWriteBasket()==0 && basket->GetNevBuf()==0) {
+ // In older files, if the branch is a TBranchElement non-terminal 'object' branch, it's basket will contain 0
+ // events, in newer file in the same case, the write basket will be missing.
+ if (from->GetEntries()!=0 && from->GetWriteBasket()==0 && (basket==0 || basket->GetNevBuf()==0)) {
to->SetEntries(to->GetEntries()+from->GetEntries());
}
}
@@ -449,22 +429,25 @@ void TTreeCloner::WriteBaskets()
Int_t index = fBasketNum[ fBasketIndex[j] ];
Long64_t pos = from->GetBasketSeek(index);
- if (from->GetBasketBytes()[index] == 0) {
- from->GetBasketBytes()[index] = basket->ReadBasketBytes(pos, fromfile);
- }
- Int_t len = from->GetBasketBytes()[index];
- Long64_t entryInBasket;
- if (index == from->GetWriteBasket()) {
- entryInBasket = from->GetEntries() - from->GetBasketEntry()[index];
+ if (pos!=0) {
+ if (from->GetBasketBytes()[index] == 0) {
+ from->GetBasketBytes()[index] = basket->ReadBasketBytes(pos, fromfile);
+ }
+ Int_t len = from->GetBasketBytes()[index];
+
+ basket->LoadBasketBuffers(pos,len,fromfile);
+ basket->IncrementPidOffset(fPidOffset);
+ basket->CopyTo(tofile);
+ to->AddBasket(*basket,kTRUE,fToStartEntries + from->GetBasketEntry()[index]);
} else {
- entryInBasket = from->GetBasketEntry()[index+1] - from->GetBasketEntry()[index];
+ TBasket *frombasket = from->GetBasket( index );
+ if (frombasket->GetNevBuf()>0) {
+ TBasket *tobasket = (TBasket*)frombasket->Clone();
+ tobasket->SetBranch(to);
+ to->AddBasket(*tobasket, kFALSE, fToStartEntries+from->GetBasketEntry()[index]);
+ to->FlushOneBasket(to->GetWriteBasket());
+ }
}
-
- basket->LoadBasketBuffers(pos,len,fromfile);
- basket->IncrementPidOffset(fPidOffset);
- basket->CopyTo(tofile);
- to->AddBasket(*basket,kTRUE,fToStartEntries + from->GetBasketEntry()[index]);
-
}
delete basket;
}
--
GitLab