[Doxygen] Move creation of notebooks out of doxygen filter step.

[ROOT-10157]
- The doxygen filter step now only writes out that it wants a
notebook, and how it should be created, but it doesn't run the notebook.
- Later, this info is picked up by makeNotebooks.sh, and notebooks are
generated in parallel.
- Add a variable in the Makefile to choose number of processes for notebook
  conversion.

documentation/doxygen/Makefile

 
 .PHONY: filter folders mathjax js images doxygen
 
-PYTHON_EXECUTABLE ?= /usr/bin/python3
+NJOB ?= $(shell nproc)
+PYTHON_EXECUTABLE ?= python3
 export PYTHON_EXECUTABLE
 export PYSPARK_PYTHON := $(PYTHON_EXECUTABLE)
 
@@ -24,7 +25,7 @@ define MkDir
 	+@[ -d $1 ] || mkdir -p $1
 endef
 
-all: filter folders mathjax js images doxygen
+all: filter folders mathjax js images doxygen replaceCollaborationDiagrams notebooks rootWork
 
 filter:
 	`root-config --cxx` -o filter filter.cxx -std=c++14 -O2
@@ -47,6 +48,7 @@ pyzdoc:
 	$(PYTHON_EXECUTABLE) print_roofit_pyz_doctrings.py > $(DOXYGEN_PYZDOC_PATH)/_roofit.pyzdoc
 
 doxygen: filter pyzdoc
+	rm -f tutorialWorklist_py tutorialWorklist_root
 	$(call MkDir,$(DOXYGEN_OUTPUT_DIRECTORY))
 	$(call MkDir,$(DOXYGEN_EXAMPLE_PATH))
 	$(call MkDir,$(DOXYGEN_NOTEBOOK_PATH))
@@ -60,7 +62,24 @@ doxygen: filter pyzdoc
 	rm -rf listofclass.sh tmva* data* result* config* test* Roo* My* Freq*
 	rm -f Doxyfile_INPUT filter htmlfooter.html MDF.C pca.C
 	rm -f greek.gif hsumanim.gif mathsymb.gif parallelcoordtrans.gif
+	rm -rf files c1* *.ps *.png *.svg *.pdf *.root *.xpm *.out *.dat *.dtd *.dot *.txt listofclass.sh
+
+collaborationDiagrams:
+	bash ./makeCollaborationDiagrams.sh
+
+replaceCollaborationDiagrams: doxygen collaborationDiagrams
+	bash ./modifyClassWebpages.sh -j$(NJOB)
+
+tutorialWorklist_py: doxygen
+tutorialWorklist_root: doxygen
+
+notebooks: tutorialWorklist_py makeNotebooks.sh
+	cp ../../tutorials/unfold/*.xml ../../tutorials/unfold/*.dtd $(DOXYGEN_NOTEBOOK_PATH)/
+	bash ./makeNotebooks.sh $< -j$(NJOB)
 	rm -f $(DOXYGEN_NOTEBOOK_PATH)/*.root
 
+rootWork: tutorialWorklist_root
+	xargs -L 1 -P 12 root < $<
+
 clean:
-	rm -rf $(DOXYGEN_OUTPUT_DIRECTORY)
+	rm -rf $(DOXYGEN_OUTPUT_DIRECTORY) tutorialWorklist_py tutorialWorklist_root

documentation/doxygen/filter.cxx

@@ -76,6 +76,7 @@
 #include <string>
 #include <cstring>
 #include <iostream>
+#include <fstream>
 #include <stdlib.h>
 #include <stdarg.h>
 #include <memory>
@@ -300,6 +301,12 @@ void FilterClass()
 
 void FilterTutorial()
 {
+   // Use these to write out work that should be run in parallel after doxygen is done:
+   // This executes python <work>
+   std::ofstream worklist_py("tutorialWorklist_py", ios_base::app);
+   // This executes root <work>
+   std::ofstream worklist_root("tutorialWorklist_root", ios_base::app);
+
    // File for inline macros.
    FILE *m = 0;
 
@@ -400,9 +407,9 @@ void FilterTutorial()
 
       // notebook found
       if (gLineString.find("\\notebook") != string::npos) {
-         ExecuteCommand(StringFormat("%s converttonotebook.py %s %s/notebooks/",
-                                          gPythonExec.c_str(),
-                                          gFileName.c_str(), gOutDir.c_str()));
+         // Notebooks are generated in dedicated step:
+         worklist_py << "converttonotebook.py " << gFileName << " " << gOutDir << "/notebooks/" << std::endl;
+
          if (gPython){
              gLineString = "## ";
          }

documentation/doxygen/makeNotebooks.sh

+#!/bin/bash
+# Run commands to generate notebooks, possibly in parallel.
+# List dependencies below as <tutorial> <tutorial it depends on>
+
+
+while [ $# -gt 0 ]; do
+  case $1 in
+    -j*)
+      nJobs=${1#-j}
+      shift
+      ;;
+    *)
+      inputFile=$1
+      shift
+      ;;
+  esac
+done
+
+if [ ! -f "$inputFile" ]; then
+  echo "Usage: makeNotebooks <fileWithNotebooks to generate> [-j jobs]"
+  exit 1
+fi
+
+# Prepare jupyter runs
+mkdir -p ~/.jupyter
+
+# If notebooks depend on the output of others, run those first.
+# List dependencies below in the form
+# <Notebook with dependencies> <dependency> [<dependency> ...]
+while read notebook dependencies; do
+	if grep -q ${notebook} $inputFile; then
+    for dependency in $dependencies; do
+      cmd=$(grep $dependency $inputFile)
+      if [ -n "$cmd" ]; then
+        echo "Running $cmd as depedency of $notebook"
+        ${PYTHON_EXECUTABLE:-python3} $cmd  && sed -i'.back' "\#${dependency}#d" $inputFile
+      fi
+    done
+  fi
+done <<EOF
+math/testUnfold5d.C   math/testUnfold5a.C math/testUnfold5b.C math/testUnfold5c.C
+xml/xmlreadfile.C   xml/xmlnewfile.C
+roofit/rf503_wspaceread.C   roofit/rf502_wspacewrite.C roofit/rf502_wspacewrite.py
+io/readCode.C   io/importCode.C
+fit/fit1.C   hist/fillrandom.C
+fit/myfit.C   fit/fitslicesy.C
+foam/foam_demopers.C   foam/foam_demo.C
+tree/staff.C   tree/cernbuild.C
+tree/cernstaff.C   tree/cernbuild.C
+hist/hbars.C   tree/cernbuild.C
+pyroot/ntuple1.py   pyroot/hsimple.py
+pyroot/h1draw.py   pyroot/hsimple.py
+pyroot/fit1.py   pyroot/fillrandom.py
+tmva/TMVAClassificationApplication.C   tmva/TMVAClassification.C
+tmva/TMVAClassificationCategory.C   tmva/TMVAClassification.C
+tmva/TMVAClassificationCategoryApplication.C   tmva/TMVAClassificationCategory.C
+tmva/TMVAMulticlass.C   tmva/TMVAMultipleBackgroundExample.C
+tmva/TMVAMulticlassApplication.C   tmva/TMVAMulticlass.C
+tmva/TMVARegressionApplication.C   tmva/TMVARegression.C
+EOF
+
+# Run rest in parallel
+xargs -L 1 -P ${nJobs:-1} ${PYTHON_EXECUTABLE:-python3} < $inputFile
+
+rm ${inputFile}.back
+