[forest] WIP - 32bit offset columns / cluster support

tree/forest/v7/inc/ROOT/RColumn.hxx

@@ -149,10 +149,10 @@ public:
 
    /// For offset columns only, do index arithmetic from cluster-local to global indizes
    void GetCollectionInfo(const ForestSize_t index, ForestSize_t* collectionStart, ClusterSize_t* collectionSize) {
-      ForestSize_t dummy;
-      RColumnElement<ForestSize_t, EColumnType::kIndex> fElemDummy(&dummy);
-      auto idxStart = (index == 0) ? 0 : *Map<ForestSize_t, EColumnType::kIndex>(index - 1, &fElemDummy);
-      auto idxEnd = *Map<ForestSize_t, EColumnType::kIndex>(index, &fElemDummy);
+      ClusterSize_t dummy;
+      RColumnElement<ClusterSize_t, EColumnType::kIndex> fElemDummy(&dummy);
+      auto idxStart = (index == 0) ? 0 : *Map<ClusterSize_t, EColumnType::kIndex>(index - 1, &fElemDummy);
+      auto idxEnd = *Map<ClusterSize_t, EColumnType::kIndex>(index, &fElemDummy);
       auto selfOffset = fCurrentPage.GetClusterInfo().GetSelfOffset();
       auto pointeeOffset = fCurrentPage.GetClusterInfo().GetPointeeOffset();
       if (index == selfOffset) {

tree/forest/v7/inc/ROOT/RColumnElement.hxx

@@ -144,12 +144,12 @@ public:
 
 template <>
 class ROOT::Experimental::Detail::RColumnElement<
-   ROOT::Experimental::ForestSize_t, ROOT::Experimental::EColumnType::kIndex>
+   ROOT::Experimental::ClusterSize_t, ROOT::Experimental::EColumnType::kIndex>
    : public ROOT::Experimental::Detail::RColumnElementBase {
 public:
    static constexpr bool kIsMappable = true;
-   static constexpr size_t kSize = sizeof(ROOT::Experimental::ForestSize_t);
-   explicit RColumnElement(ROOT::Experimental::ForestSize_t* value) : RColumnElementBase(value, kSize, kIsMappable) {}
+   static constexpr size_t kSize = sizeof(ROOT::Experimental::ClusterSize_t);
+   explicit RColumnElement(ROOT::Experimental::ClusterSize_t* value) : RColumnElementBase(value, kSize, kIsMappable) {}
 };
 
 template <>

tree/forest/v7/inc/ROOT/RColumnModel.hxx

@@ -56,7 +56,7 @@ enum class EColumnType {
  * Lookup table for the element size in bytes for column types. The array has to correspond to EColumnTypes.
  */
 constexpr std::size_t kColumnElementSizes[] =
-  {0 /* kUnknown */, 8 /* kIndex */, 1 /* kByte */, 8 /* kReal64 */, 4 /* kReal32 */, 2 /* kReal16 */,
+  {0 /* kUnknown */, 4 /* kIndex */, 1 /* kByte */, 8 /* kReal64 */, 4 /* kReal32 */, 2 /* kReal16 */,
    1 /* kReal8 */, 8 /* kInt64 */, 4 /* kInt32 */, 2 /* kInt16 */};
 
 // clang-format off

tree/forest/v7/inc/ROOT/RField.hxx

@@ -253,7 +253,7 @@ public:
 class RFieldVector : public Detail::RFieldBase {
 private:
    size_t fItemSize;
-   ForestSize_t fNWritten;
+   ClusterSize_t fNWritten;
 
 protected:
    void DoAppend(const Detail::RFieldValueBase& value) final;
@@ -273,6 +273,7 @@ public:
    void DestroyValue(const Detail::RFieldValueBase& value, bool dtorOnly = false) final;
    Detail::RFieldValueBase CaptureValue(void *where) override;
    size_t GetValueSize() const override;
+   void CommitCluster() final;
 };
 
 
@@ -312,14 +313,14 @@ public:
 
    using Detail::RFieldBase::GenerateValue;
    ROOT::Experimental::Detail::RFieldValueBase GenerateValue(void* where) final {
-      return ROOT::Experimental::RFieldValue<ForestSize_t>(
-         Detail::RColumnElement<ForestSize_t, EColumnType::kIndex>(static_cast<ForestSize_t*>(where)),
-         this, static_cast<ForestSize_t*>(where));
+      return ROOT::Experimental::RFieldValue<ClusterSize_t>(
+         Detail::RColumnElement<ClusterSize_t, EColumnType::kIndex>(static_cast<ClusterSize_t*>(where)),
+         this, static_cast<ClusterSize_t*>(where));
    }
    Detail::RFieldValueBase CaptureValue(void* where) final {
-      return ROOT::Experimental::RFieldValue<ForestSize_t>(true,
-         Detail::RColumnElement<ForestSize_t, EColumnType::kIndex>(static_cast<ForestSize_t*>(where)),
-         this, static_cast<ForestSize_t*>(where));
+      return ROOT::Experimental::RFieldValue<ClusterSize_t>(true,
+         Detail::RColumnElement<ClusterSize_t, EColumnType::kIndex>(static_cast<ClusterSize_t*>(where)),
+         this, static_cast<ClusterSize_t*>(where));
    }
    size_t GetValueSize() const final { return 0; }
 };
@@ -329,9 +330,9 @@ public:
 
 
 template <>
-class ROOT::Experimental::RField<ROOT::Experimental::ForestSize_t> : public ROOT::Experimental::Detail::RFieldBase {
+class ROOT::Experimental::RField<ROOT::Experimental::ClusterSize_t> : public ROOT::Experimental::Detail::RFieldBase {
 public:
-   static std::string MyTypeName() { return "ROOT::Experimental::ForestSize_t"; }
+   static std::string MyTypeName() { return "ROOT::Experimental::ClusterSize_t"; }
    explicit RField(std::string_view name) : Detail::RFieldBase(name, MyTypeName(), true /* isSimple */) {}
    RField(RField&& other) = default;
    RField& operator =(RField&& other) = default;
@@ -341,29 +342,34 @@ public:
    void DoGenerateColumns() final;
    unsigned int GetNColumns() const final { return 1; }
 
-   ForestSize_t* Map(ForestSize_t index) {
-      static_assert(Detail::RColumnElement<ForestSize_t, EColumnType::kIndex>::kIsMappable,
-                    "(ForestSize_t, EColumnType::kIndex) is not identical on this platform");
-      return fPrincipalColumn->Map<ForestSize_t, EColumnType::kIndex>(index, nullptr);
+   ClusterSize_t* Map(ForestSize_t index) {
+      static_assert(Detail::RColumnElement<ClusterSize_t, EColumnType::kIndex>::kIsMappable,
+                    "(ClusterSize_t, EColumnType::kIndex) is not identical on this platform");
+      return fPrincipalColumn->Map<ClusterSize_t, EColumnType::kIndex>(index, nullptr);
    }
 
    using Detail::RFieldBase::GenerateValue;
    template <typename... ArgsT>
    ROOT::Experimental::Detail::RFieldValueBase GenerateValue(void* where, ArgsT&&... args)
    {
-      ROOT::Experimental::RFieldValue<ForestSize_t> v(
-         Detail::RColumnElement<ForestSize_t, EColumnType::kIndex>(static_cast<ForestSize_t*>(where)),
-         this, static_cast<ForestSize_t*>(where), std::forward<ArgsT>(args)...);
+      ROOT::Experimental::RFieldValue<ClusterSize_t> v(
+         Detail::RColumnElement<ClusterSize_t, EColumnType::kIndex>(static_cast<ClusterSize_t*>(where)),
+         this, static_cast<ClusterSize_t*>(where), std::forward<ArgsT>(args)...);
       return v;
    }
    ROOT::Experimental::Detail::RFieldValueBase GenerateValue(void* where) final { return GenerateValue(where, 0); }
    Detail::RFieldValueBase CaptureValue(void *where) final {
-      ROOT::Experimental::RFieldValue<ForestSize_t> v(true,
-         Detail::RColumnElement<ForestSize_t, EColumnType::kIndex>(static_cast<ForestSize_t*>(where)),
-         this, static_cast<ForestSize_t*>(where));
+      ROOT::Experimental::RFieldValue<ClusterSize_t> v(true,
+         Detail::RColumnElement<ClusterSize_t, EColumnType::kIndex>(static_cast<ClusterSize_t*>(where)),
+         this, static_cast<ClusterSize_t*>(where));
       return v;
    }
-   size_t GetValueSize() const final { return sizeof(ForestSize_t); }
+   size_t GetValueSize() const final { return sizeof(ClusterSize_t); }
+
+   /// Special help for offset fields
+   void GetCollectionInfo(ForestSize_t index, ForestSize_t* idxStart, ClusterSize_t* size) {
+      fPrincipalColumn->GetCollectionInfo(index, idxStart, size);
+   }
 };
 
 
@@ -489,8 +495,8 @@ public:
 template <>
 class ROOT::Experimental::RField<std::string> : public ROOT::Experimental::Detail::RFieldBase {
 private:
-   ForestSize_t fIndex;
-   Detail::RColumnElement<ForestSize_t, EColumnType::kIndex> fElemIndex;
+   ClusterSize_t fIndex;
+   Detail::RColumnElement<ClusterSize_t, EColumnType::kIndex> fElemIndex;
 
    void DoAppend(const ROOT::Experimental::Detail::RFieldValueBase& value) final;
    void DoRead(ROOT::Experimental::ForestSize_t index, ROOT::Experimental::Detail::RFieldValueBase* value) final;

tree/forest/v7/inc/ROOT/RForestView.hxx

@@ -141,7 +141,7 @@ public:
 \brief A tree view for a collection, that can itself generate new tree views for its nested fields.
 */
 // clang-format on
-class RForestViewCollection : public RForestView<ForestSize_t> {
+class RForestViewCollection : public RForestView<ClusterSize_t> {
     friend class RInputForest;
 
 private:
@@ -149,7 +149,7 @@ private:
    Detail::RPageSource* fSource;
 
    RForestViewCollection(std::string_view fieldName, Detail::RPageSource* source)
-      : RForestView<ForestSize_t>(fieldName, source)
+      : RForestView<ClusterSize_t>(fieldName, source)
       , fCollectionName(fieldName)
       , fSource(source)
    {}
@@ -168,8 +168,10 @@ public:
    ~RForestViewCollection() = default;
 
    RForestViewRange GetViewRange(ForestSize_t index) {
-      ForestSize_t start = (index == 0) ? 0 : *fField.Map(index - 1);
-      return RForestViewRange(start, *fField.Map(index));
+      ClusterSize_t size;
+      ForestSize_t idxStart;
+      fField.GetCollectionInfo(index, &idxStart, &size);
+      return RForestViewRange(idxStart, idxStart + size);
    }
    template <typename T>
    RForestView<T> GetView(std::string_view fieldName) { return RForestView<T>(GetSubName(fieldName), fSource); }
@@ -177,9 +179,11 @@ public:
       return RForestViewCollection(GetSubName(fieldName), fSource);
    }
 
-   ForestSize_t operator()(ForestSize_t index) {
-      ForestSize_t offsetPrev = (index == 0) ? 0 : *fField.Map(index - 1);
-      return *fField.Map(index) - offsetPrev;
+   ClusterSize_t operator()(ForestSize_t index) {
+      ClusterSize_t size;
+      ForestSize_t idxStart;
+      fField.GetCollectionInfo(index, &idxStart, &size);
+      return size;
    }
 };
 

tree/forest/v7/src/RField.cxx

@@ -50,7 +50,7 @@ ROOT::Experimental::Detail::RFieldBase::Create(const std::string &fieldName, con
    if (normalizedType == "string") normalizedType = "std::string";
    if (normalizedType.substr(0, 7) == "vector<") normalizedType = "std::" + normalizedType;
 
-   if (normalizedType == "ROOT::Experimental::ForestSize_t") return new RField<ForestSize_t>(fieldName);
+   if (normalizedType == "ROOT::Experimental::ClusterSize_t") return new RField<ClusterSize_t>(fieldName);
    if (normalizedType == "std::uint32_t") return new RField<std::uint32_t>(fieldName);
    if (normalizedType == "float") return new RField<float>(fieldName);
    if (normalizedType == "double") return new RField<double>(fieldName);
@@ -61,7 +61,7 @@ ROOT::Experimental::Detail::RFieldBase::Create(const std::string &fieldName, con
       return new RFieldVector(fieldName, std::unique_ptr<Detail::RFieldBase>(itemField));
    }
    // TODO: create an RFieldCollection?
-   if (normalizedType == ":Collection:") return new RField<ForestSize_t>(fieldName);
+   if (normalizedType == ":Collection:") return new RField<ClusterSize_t>(fieldName);
    auto cl = TClass::GetClass(normalizedType.c_str());
    if (cl != nullptr) {
       return new RFieldClass(fieldName, normalizedType);
@@ -205,7 +205,7 @@ ROOT::Experimental::RForestEntry* ROOT::Experimental::RFieldRoot::GenerateEntry(
 //------------------------------------------------------------------------------
 
 
-void ROOT::Experimental::RField<ROOT::Experimental::ForestSize_t>::DoGenerateColumns()
+void ROOT::Experimental::RField<ROOT::Experimental::ClusterSize_t>::DoGenerateColumns()
 {
    RColumnModel model(GetName(), EColumnType::kIndex, true /* isSorted*/);
    fColumns.emplace_back(std::make_unique<Detail::RColumn>(model));
@@ -389,7 +389,7 @@ void ROOT::Experimental::RFieldVector::DoAppend(const Detail::RFieldValueBase& v
       auto itemValue = fSubFields[0]->CaptureValue(typedValue->data() + (i * fItemSize));
       fSubFields[0]->Append(itemValue);
    }
-   Detail::RColumnElement<ForestSize_t, EColumnType::kIndex> elemIndex(&fNWritten);
+   Detail::RColumnElement<ClusterSize_t, EColumnType::kIndex> elemIndex(&fNWritten);
    fNWritten += count;
    fColumns[0]->Append(elemIndex);
 }
@@ -397,12 +397,9 @@ void ROOT::Experimental::RFieldVector::DoAppend(const Detail::RFieldValueBase& v
 void ROOT::Experimental::RFieldVector::DoRead(ForestSize_t index, Detail::RFieldValueBase* value) {
    auto typedValue = reinterpret_cast<RFieldValue<std::vector<char>>*>(value)->Get();
 
-   ForestSize_t dummy;
-   Detail::RColumnElement<ForestSize_t, EColumnType::kIndex> elemIndex(&dummy);
-   auto idxStart = (index == 0) ? 0
-      : *fColumns[0]->template Map<ForestSize_t, EColumnType::kIndex>(index - 1, &elemIndex);
-   auto idxEnd = *fColumns[0]->template Map<ForestSize_t, EColumnType::kIndex>(index, &elemIndex);
-   auto nItems = idxEnd - idxStart;
+   ClusterSize_t nItems;
+   ForestSize_t idxStart;
+   fPrincipalColumn->GetCollectionInfo(index, &idxStart, &nItems);
 
    typedValue->resize(nItems * fItemSize);
    for (unsigned i = 0; i < nItems; ++i) {
@@ -453,6 +450,11 @@ size_t ROOT::Experimental::RFieldVector::GetValueSize() const
    return sizeof(std::vector<char>);
 }
 
+void ROOT::Experimental::RFieldVector::CommitCluster()
+{
+   fNWritten = 0;
+}
+
 
 //------------------------------------------------------------------------------
 

tree/forest/v7/test/forest.cxx

@@ -202,7 +202,6 @@ TEST(RForest, Clusters)
 
 TEST(RForest, View)
 {
-   // TODO(jblomer): test with multiple clusters
    auto model = RForestModel::Create();
    auto fieldPt = model->MakeField<float>("pt", 42.0);
    auto fieldTag = model->MakeField<std::string>("tag", "xyz");
@@ -213,6 +212,9 @@ TEST(RForest, View)
    {
       ROutputForest forest(std::move(model), std::make_unique<RPageSinkRoot>("f", "test.root"));
       forest.Fill();
+      forest.CommitCluster();
+      fieldJets->clear();
+      forest.Fill();
    }
 
    RInputForest forest(std::make_unique<RPageSourceRoot>("f", "test.root"));
@@ -222,17 +224,21 @@ TEST(RForest, View)
       EXPECT_EQ(42.0, viewPt(i));
       n++;
    }
-   EXPECT_EQ(1, n);
+   EXPECT_EQ(2, n);
 
    auto viewJets = forest.GetView<std::vector<float>>("jets");
    n = 0;
    for (auto i : forest.GetViewRange()) {
-      EXPECT_EQ(2U, viewJets(i).size());
-      EXPECT_EQ(1.0, viewJets(i)[0]);
-      EXPECT_EQ(2.0, viewJets(i)[1]);
+      if (i == 0) {
+         EXPECT_EQ(2U, viewJets(i).size());
+         EXPECT_EQ(1.0, viewJets(i)[0]);
+         EXPECT_EQ(2.0, viewJets(i)[1]);
+      } else {
+         EXPECT_EQ(0U, viewJets(i).size());
+      }
       n++;
    }
-   EXPECT_EQ(1, n);
+   EXPECT_EQ(2, n);
 }
 
 TEST(RForest, Capture) {
@@ -270,6 +276,8 @@ TEST(RForest, Composable)
             fldTracks->Fill();
          }
          *fldPt = float(i);
+         //if (i == 2)
+         //   forest->CommitCluster();
          forest->Fill();
       }
    }