Build pch with python script for windows compatibility

the script has been refactored.
All *nix traditional programs like dirname, awk, ... have been replaced
with python functionality.

CMakeLists.txt

@@ -133,8 +133,8 @@ get_property(__clingetcpch GLOBAL PROPERTY CLINGETCPCH)
 add_custom_command(OUTPUT etc/dictpch/allLinkDefs.h
                           etc/dictpch/allHeaders.h
                           etc/dictpch/allCppflags.txt
-                   COMMAND ${CMAKE_SOURCE_DIR}/build/unix/makepchinput.sh ${CMAKE_SOURCE_DIR} "." ${__clingetcpch}
-                   DEPENDS ${CMAKE_SOURCE_DIR}/build/unix/makepchinput.sh ${__allFiles})
+                   COMMAND ${CMAKE_SOURCE_DIR}/build/unix/makepchinput.py ${CMAKE_SOURCE_DIR} "." ${__clingetcpch}
+                   DEPENDS ${CMAKE_SOURCE_DIR}/build/unix/makepchinput.py ${__allFiles})
 
 add_custom_command(OUTPUT etc/allDict.cxx.pch
                    COMMAND ${CMAKE_SOURCE_DIR}/etc/dictpch/makepch.sh etc/allDict.cxx.pch -I${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}

build/unix/makepchinput.py

+#! /usr/bin/env python
+#
+# Extract the input needed to build a PCH for the main (enabled) ROOT modules.
+# Script takes as argument the source directory path, the set of enabled
+# modules and extra headers (from cling) to be included in the PCH.
+#
+# Copyright (c) 2014 Rene Brun and Fons Rademakers
+# Author: Axel Naumann <axel@cern.ch>, 2014-10-16
+# Translated to Python by Danilo Piparo, 2015-04-22
+
+import sys
+import os
+import glob
+import shutil
+from sets import Set
+
+#-------------------------------------------------------------------------------
+def removeFiles(filesList):
+   existingFilesList = filter(os.path.exists,filesList)
+   map(os.unlink, existingFilesList)
+
+#-------------------------------------------------------------------------------
+def removeLeftOvers():
+   """
+   Remove leftover files from old versions of this script.
+   """
+   filesToRemove = [os.path.join("include","allHeaders.h"),
+                    os.path.join("include","allHeaders.h.pch"),
+                    os.path.join("include","allLinkDef.h"),
+                    "all.h",
+                    "cppflags.txt",
+                    os.path.join("include","allLinkDef.h"),
+                    os.path.join("etc","allDict.cxx"),
+                    os.path.join("etc","allDict.cxx.h")]
+   removeFiles(filesToRemove)
+
+#-------------------------------------------------------------------------------
+def getParams():
+   """
+   Extract parameters from the commandline, which looks like
+   makePCHInput.py ZZZ XXX YYY
+   """
+   argv = sys.argv
+   rootSrcDir, modules = argv[1:3]
+   clingetpchList = argv[3:]
+   return rootSrcDir, modules, clingetpchList
+
+#-------------------------------------------------------------------------------
+def getGuardedStlInclude(headerName):
+   return '#if __has_include("%s")\n' %headerName +\
+          '#include <%s>\n' %headerName +\
+          '#endif\n'
+
+#-------------------------------------------------------------------------------
+def getSTLIncludes():
+   """
+   Here we include the list of c++11 stl headers
+   From http://en.cppreference.com/w/cpp/header
+   regex is removed until ROOT-7004 is fixed
+   """
+   stlHeadersList = ("cstdlib",
+                     "csignal",
+                     "csetjmp",
+                     "cstdarg",
+                     "typeinfo",
+                     "typeindex",
+                     "type_traits",
+                     "bitset",
+                     "functional",
+                     "utility",
+                     "ctime",
+                     "chrono",
+                     "cstddef",
+                     "initializer_list",
+                     "tuple",
+                     "new",
+                     "memory",
+                     "scoped_allocator",
+                     "climits",
+                     "cfloat",
+                     "cstdint",
+                     "cinttypes",
+                     "limits",
+                     "exception",
+                     "stdexcept",
+                     "cassert",
+                     "system_error",
+                     "cerrno",
+                     "cctype",
+                     "cwctype",
+                     "cstring",
+                     "cwchar",
+                     "cuchar",
+                     "string",
+                     "array",
+                     "vector",
+                     "deque",
+                     "list",
+                     "forward_list",
+                     "set",
+                     "map",
+                     "unordered_set",
+                     "unordered_map",
+                     "stack",
+                     "queue",
+                     "algorithm",
+                     "iterator",
+                     "cmath",
+                     "complex",
+                     "valarray",
+                     "random",
+                     "numeric",
+                     "ratio",
+                     "cfenv",
+                     "iosfwd",
+                     "ios",
+                     "istream",
+                     "ostream",
+                     "iostream",
+                     "fstream",
+                     "sstream",
+                     "iomanip",
+                     "streambuf",
+                     "cstdio",
+                     "locale",
+                     "clocale",
+                     "codecvt",
+                     "atomic",
+                     "thread",
+                     "mutex",
+                     "future",
+                     "condition_variable",
+                     "ciso646",
+                     "ccomplex",
+                     "ctgmath",
+                     "cstdalign",
+                     "cstdbool")
+
+   allHeadersPartContent = "// STL headers\n"
+
+   for header in stlHeadersList:
+      allHeadersPartContent += getGuardedStlInclude(header)
+
+   # Special case for regex
+   allHeadersPartContent += '// treat regex separately\n' +\
+                            '#if __has_include("regex") && !defined __APPLE__\n' +\
+                            '#include <regex>\n' +\
+                            '#endif\n'
+
+   # treat this deprecated headers in a special way
+   stlDeprecatedHeadersList=["strstream"]
+   allHeadersPartContent += '// STL Deprecated headers\n' +\
+                            '#define _BACKWARD_BACKWARD_WARNING_H\n' +\
+                            "#pragma clang diagnostic push\n" +\
+                            '#pragma GCC diagnostic ignored "-Wdeprecated"\n'
+
+   for header in stlDeprecatedHeadersList:
+      allHeadersPartContent += getGuardedStlInclude(header)
+
+   allHeadersPartContent += '#pragma clang diagnostic pop\n' +\
+                            '#undef _BACKWARD_BACKWARD_WARNING_H\n'
+   return allHeadersPartContent
+
+#-------------------------------------------------------------------------------
+def getExtraIncludes(headers):
+   """
+   Add include files according to list
+   """
+   allHeadersPartContent=""
+   for header in headers:
+      allHeadersPartContent+='#include "%s"\n' %header
+   return allHeadersPartContent
+
+#-------------------------------------------------------------------------------
+def getDictNames(theDirName):
+   """
+   Get a list of all dictionaries in a directory
+   """
+   #`find $modules -name 'G__*.cxx' 2> /dev/null | grep -v /G__Cling.cxx  | grep -v core/metautils/src/G__std_`; do
+   wildcards = (os.path.join(theDirName , "*", "*", "G__*.cxx"),
+                os.path.join(theDirName , "*", "G__*.cxx"))
+   allDictNames = []
+   for wildcard in wildcards:
+      allDictNames += glob.glob(wildcard)
+   stdDictpattern = os.path.join("core","metautils","src","G__std_")
+   dictNames = filter (lambda dictName: not ('G__Cling.cxx' in dictName or stdDictpattern in dictName),allDictNames )
+   return dictNames
+
+#-------------------------------------------------------------------------------
+def getDirName(dictName):
+   """
+   foo/src/G__PIPPO.cxx --> foo/
+   """
+   # get rid of the drive on win
+   dirName = os.path.splitdrive(dictName)
+
+   # foo/src/G__PIPPO.cxx --> foo/src/
+   dirName = os.path.split(dictName)[0]
+
+   # foo/src/ --> foo/src
+   dirName = dirName[:-1]
+
+   # foo/src --> foo/
+   return os.path.normpath(os.path.split(dictName)[0])
+
+#-------------------------------------------------------------------------------
+def isAnyPatternInString(patterns,theString):
+   """
+   Check if any of the patterns is contained in the string
+   """
+   for pattern in patterns:
+      if pattern in theString: return True
+   return False
+
+#-------------------------------------------------------------------------------
+def isDirForPCH(dirName):
+   """
+   Check if the directory corresponds to a module whose headers must belong to
+   the PCH
+   """
+   PCHPatternsWhitelist = ("interpreter/",
+                           "core/",
+                           "io/io",
+                           "net/net",
+                           "math/",
+                           "hist/",
+                           "tree/",
+                           "graf2d",
+                           "graf3d/ftgl",
+                           "graf3d/g3d",
+                           "graf3d/gl",
+                           "gui/gui",
+                           "gui/fitpanel",
+                           "rootx",
+                           "bindings/pyroot",
+                           "roofit/",
+                           "tmva",
+                           "main")
+   PCHPatternsBlacklist = ("graf2d/qt",
+                           "gui/guihtml",
+                           "gui/guibuilder",
+                           "math/fftw",
+                           "math/foam",
+                           "math/fumili",
+                           "math/mlp",
+                           "math/quadp",
+                           "math/splot",
+                           "math/unuran",
+                           "math/vc",
+                           "math/vdt")
+
+   accepted = isAnyPatternInString(PCHPatternsWhitelist,dirName) and \
+               not isAnyPatternInString(PCHPatternsBlacklist,dirName)
+
+   return accepted
+
+#-------------------------------------------------------------------------------
+def getLinesFromDict(marker, dictFileName):
+   """
+   Search for the line marker
+   in the dictionary and save all lines until the line '0'
+   Return them as List
+   """
+   selectedLines = []
+   ifile = open(dictFileName)
+   lines = ifile.readlines()
+   ifile.close()
+   recording = False
+   for line in lines:
+      if marker in line:
+         recording = True
+         continue
+
+      if recording and "0" == line[0]: break
+
+      if recording:
+         selectedLines.append(line[:-1])
+
+   return selectedLines
+
+#-------------------------------------------------------------------------------
+def getIncludeLinesFromDict(dictFileName):
+   """
+   Search for the headers after the line
+   'static const char* headers[]'
+   Return the code to be added to the allHeaders as string
+   """
+   allHeadersPartContent=""
+   selectedLines = getLinesFromDict('static const char* headers[]', dictFileName)
+   allHeadersPartContent += "// %s\n"  % dictFileName
+   for selectedLine in selectedLines:
+      header = selectedLine[:-1] # remove the ","
+      allHeadersPartContent += "#include %s\n" %header
+   return allHeadersPartContent
+
+#-------------------------------------------------------------------------------
+def getIncludePathsFromDict(dictFileName):
+   """
+   Search for the headers after the line
+   'static const char* headers[]'
+   Return them as list
+   """
+   incPathsPart=[]
+   selectedLines = getLinesFromDict('static const char* includePaths[]', dictFileName)
+   for selectedLine in selectedLines:
+      incPath = selectedLine[1:-2] # remove the "," and the two '"'
+      incPathsPart.append(incPath)
+   return incPathsPart
+
+#-------------------------------------------------------------------------------
+def getDefUndefLines(dirName):
+   """
+   Add undefines and defines if the directory needs them
+   """
+   allHeadersPartContent=""
+   if "%sqt" %os.sep in dirName:
+      allHeadersPartContent += '#ifdef emit\n' +\
+                               '# undef emit\n' +\
+                               '#endif\n' +\
+                               '#ifdef signals\n' +\
+                               '# undef signals\n' +\
+                               '#endif\n'
+   if "%snet%sldap" %(os.sep,os.sep) in dirName:
+      allHeadersPartContent += '#ifdef Debug\n' +\
+                               '# undef Debug\n' +\
+                               '#endif\n' +\
+                               '#ifdef GSL_SUCCESS\n' +\
+                               '# undef GSL_SUCCESS\n' +\
+                               '#endif\n'
+   return allHeadersPartContent
+
+#-------------------------------------------------------------------------------
+def mkdirIfNotThere(dirName):
+   if not os.path.exists(dirName):
+      os.mkdir(dirName)
+
+#-------------------------------------------------------------------------------
+def copyLinkDefs(rootSrcDir, outdir , dirName):
+   """
+   Extract the linkdef files
+   """
+   linkDefPartContent = ""
+   curDir = os.getcwd()
+   os.chdir(rootSrcDir)
+   wildcard = os.path.join(dirName, "inc", "*LinkDef*.h")
+   linkDefNames = glob.glob(wildcard)
+   os.chdir(curDir)
+   for linkDefName in linkDefNames:
+      linkDefDirName = os.path.basename(linkDefName)
+      mkdirIfNotThere(os.path.join(outdir,linkDefDirName))
+      srcName = os.path.join(rootSrcDir,linkDefName)
+      destName = os.path.join(outdir,linkDefName)
+      shutil.copyfile(srcName, destName)
+
+#-------------------------------------------------------------------------------
+def getLocalLinkDefs(rootSrcDir, outdir , dirName):
+   linkDefPartContent = ""
+   curDir = os.getcwd()
+   os.chdir(rootSrcDir)
+   wildcards = (os.path.join(dirName , "*", "*", "*LinkDef*.h"),
+                os.path.join(dirName , "*", "*LinkDef*.h"))
+   linkDefNames = []
+   for wildcard in wildcards:
+      linkDefNames += glob.glob(wildcard)
+
+   # now get the ones in the inc directory
+   linkDefNames = filter (lambda name: "%sinc%s" %(os.sep,os.sep) in name, linkDefNames)
+
+   for linkDefName in linkDefNames:
+      fullLinkDefName = os.path.join(outdir,linkDefName)
+      linkDefPartContent += '#include "%s"\n' %fullLinkDefName
+   os.chdir(curDir)
+   return linkDefPartContent
+
+#-------------------------------------------------------------------------------
+def resolveSoftLinks(thePaths):
+   return map(os.path.realpath,thePaths)
+
+#-------------------------------------------------------------------------------
+def getCppFlags(rootSrcDir,allIncPaths):
+   """
+   Sort, clean, no duplicates
+   cat $cppflags.tmp | sort | uniq | grep -v $srcdir | grep -v `pwd` > $cppflags
+   We must resolve softlinks.
+   returns a string
+   """
+   allHeadersPartContent = ""
+   filteredIncPaths = sorted(list(set(resolveSoftLinks(allIncPaths))))
+   for name in resolveSoftLinks((rootSrcDir,os.getcwd())):
+      filteredIncPaths = filter (lambda incPath: not name in incPath,filteredIncPaths)
+   for incPath in filteredIncPaths:
+      allHeadersPartContent += "-I%s\n" %incPath
+   return allHeadersPartContent
+
+#-------------------------------------------------------------------------------
+def writeToFile(content, filename):
+   ofile = open(filename, "w")
+   ofile.write(content)
+   ofile.close()
+
+#-------------------------------------------------------------------------------
+def writeFiles(contentFileNamePairs):
+   for content, filename in contentFileNamePairs:
+      writeToFile(content, filename)
+
+#-------------------------------------------------------------------------------
+def printModulesMessageOnScreen(selModules):
+   modulesList = sorted(list(selModules))
+   print "\nGenerating PCH for %s\n" %" ".join(modulesList)
+
+#-------------------------------------------------------------------------------
+def makePCHInput():
+   """
+   Create the input for the pch file, i.e. 3 files:
+      * etc/dictpch/allLinkDefs.h
+      * etc/dictpch/allHeaders.h
+      * etc/dictpch/allCppflags.txt
+   """
+   rootSrcDir, modules, clingetpchList = getParams()
+
+   removeLeftOvers()
+
+   outdir = os.path.join("etc","dictpch")
+   allHeadersFilename = os.path.join(outdir,"allHeaders.h")
+   allLinkdefsFilename = os.path.join(outdir,"allLinkDefs.h")
+   cppFlagsFilename = os.path.join(outdir, "allCppflags.txt")
+
+   mkdirIfNotThere(outdir)
+   removeFiles((allHeadersFilename,allLinkdefsFilename))
+
+   allHeadersContent = getSTLIncludes()
+   allHeadersContent += getExtraIncludes(clingetpchList)
+
+   allLinkdefsContent = ""
+
+   # Loop over the dictionaries, ROOT modules
+   dictNames = getDictNames(modules)
+   selModules = Set([])
+   allIncPathsList = []
+   for dictName in dictNames:
+      dirName = getDirName(dictName)
+      if not isDirForPCH(dirName): continue
+
+      selModules.add(dirName)
+
+      allHeadersContent += getIncludeLinesFromDict(dictName)
+      allIncPathsList += getIncludePathsFromDict(dictName)
+
+      allHeadersContent += getDefUndefLines(dictName)
+
+      allLinkdefsContent += getLocalLinkDefs(rootSrcDir, outdir , dirName)
+
+      copyLinkDefs(rootSrcDir, outdir , dirName)
+
+   cppFlagsContent = getCppFlags(rootSrcDir,allIncPathsList)
+
+   writeFiles(((allHeadersContent, allHeadersFilename),
+               (allLinkdefsContent, allLinkdefsFilename),
+               (cppFlagsContent, cppFlagsFilename)))
+
+
+   printModulesMessageOnScreen(selModules)
+
+if __name__ == "__main__":
+   makePCHInput()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

build/unix/makepchinput.sh

-#!/bin/sh
-#
-# Extract the input needed to build a PCH for the main (enabled) ROOT modules.
-# Script takes as argument the source directory path, the set of enabled
-# modules and extra headers (from cling) to be included in the PCH.
-#
-# Copyright (c) 2014 Rene Brun and Fons Rademakers
-# Author: Axel Naumann <axel@cern.ch>, 2014-10-16
-
-srcdir=$1
-shift
-modules=$1
-shift
-
-# Remove leftover files from old versions of this script.
-rm -f include/allHeaders.h include/allHeaders.h.pch include/allLinkDef.h all.h cppflags.txt include/allLinkDef.h etc/allDict.cxx etc/allDict.cxx.h
-
-outdir=etc/dictpch
-allheaders=$outdir/allHeaders.h
-alllinkdefs=$outdir/allLinkDefs.h
-cppflags=$outdir/allCppflags.txt
-
-mkdir -p $outdir
-rm -f $allheaders $alllinkdefs
-
-# Here we include the list of c++11 stl headers
-# From http://en.cppreference.com/w/cpp/header
-# regex is removed until ROOT-7004 is fixed
-stlHeaders="cstdlib csignal csetjmp cstdarg typeinfo typeindex type_traits bitset functional utility ctime chrono cstddef initializer_list tuple new memory scoped_allocator climits cfloat cstdint cinttypes limits exception stdexcept cassert system_error cerrno cctype cwctype cstring cwchar cuchar string array vector deque list forward_list set map unordered_set unordered_map stack queue algorithm iterator cmath complex valarray random numeric ratio cfenv iosfwd ios istream ostream iostream fstream sstream iomanip streambuf cstdio locale clocale codecvt atomic thread mutex future condition_variable ciso646 ccomplex ctgmath cstdalign cstdbool"
-
-echo "// STL headers" >> $allheaders
-for stlHeader in $stlHeaders; do
-    echo '#if __has_include("'$stlHeader'")' >> $allheaders
-    echo '#include <'$stlHeader'>' >> $allheaders
-    echo '#endif' >> $allheaders
-done
-
-# Special case for regex
-echo "// treat regex separately" >> $allheaders
-echo '#if __has_include("regex") && !defined __APPLE__' >> $allheaders
-echo '#include <regex>' >> $allheaders
-echo '#endif' >> $allheaders
-
-# treat this deprecated headers in a special way
-stlDeprecatedHeaders="strstream"
-echo "// STL Deprecated headers" >> $allheaders
-echo '#pragma clang diagnostic push' >> $allheaders
-echo '#pragma GCC diagnostic ignored "-Wdeprecated"' >> $allheaders
-for stlHeader in stlDeprecatedHeaders; do
-    echo '#if __has_include("'$stlHeader'")' >> $allheaders
-    echo '#include <'$stlHeader'>' >> $allheaders
-    echo '#endif' >> $allheaders
-done
-echo '#pragma clang diagnostic pop' >> $allheaders
-
-while ! [ "x$1" = "x" ]; do
-    echo '#include "'$1'"' >> $allheaders
-    shift
-done
-
-for dict in `find $modules -name 'G__*.cxx' 2> /dev/null | grep -v /G__Cling.cxx  | grep -v core/metautils/src/G__std_`; do
-    dirname=`dirname $dict`                   # to get foo/src
-    dirname=`echo $dirname | sed -e 's,/src$,,' -e 's,^[.]/,,' ` # to get foo/
-
-    case $dirname in
-        graf2d/qt | math/fftw | math/foam | math/fumili | math/mlp | math/quadp | math/splot | math/unuran | math/vc | math/vdt) continue;;
-
-        interpreter/* | core/* | io/io | net/net | math/* | hist/* | tree/* | graf2d/* | graf3d/ftgl | graf3d/g3d | graf3d/gl | gui/gui | gui/fitpanel | rootx | bindings/pyroot | roofit/* | tmva | main) ;;
-
-        *) continue;;
-    esac
-
-    # Check if selmodules already contains the dirname.
-    # Happens for instance for math/smatrix with its two (32bit and 64 bit)
-    # dictionaries.
-    if ! ( echo $selmodules | grep "$dirname " > /dev/null ); then
-        selmodules="$selmodules$dirname "
-    fi
-
-    awk 'BEGIN{START=-1} /includePaths\[\] = {/, /^0$/ { if (START==-1) START=NR; else if ($0 != "0") { sub(/",/,"",$0); sub(/^"/,"-I",$0); print $0 } }' $dict >> $cppflags.tmp
-    echo "// $dict" >> $allheaders
-#     awk 'BEGIN{START=-1} /payloadCode =/, /^;$/ { if (START==-1) START=NR; else if ($1 != ";") { code=substr($0,2); sub(/\\n"/,"",code); print code } }' $dict >> $allheaders
-    awk 'BEGIN{START=-1} /headers\[\] = {/, /^0$/ { if (START==-1) START=NR; else if ($0 != "0") { sub(/,/,"",$0); print "#include",$0 } }' $dict >> $allheaders
-
-    if ! test "$dirname" = "`echo $dirname| sed 's,/qt,,'`"; then
-        # something qt; undef emit afterwards
-        cat <<EOF >> $allheaders
-#ifdef emit
-# undef emit
-#endif
-#ifdef signals
-# undef signals
-#endif
-EOF
-    elif ! test "$dirname" = "`echo $dirname| sed 's,net/ldap,,'`"; then
-        # ldap; undef Debug afterwards
-        cat <<EOF >> $allheaders
-#ifdef Debug
-# undef Debug
-#endif
-#ifdef GSL_SUCCESS
-# undef GSL_SUCCESS
-#endif
-EOF
-    fi
-
-    for f in `cd $srcdir; find $dirname/inc/ -name '*LinkDef*.h'`; do
-        echo '#include "'$outdir/$f'"' >> $alllinkdefs
-    done
-done
-
-# E.g. core's LinkDef includes clib/LinkDef, so just copy all LinkDefs.
-for f in `cd $srcdir; find . -name '*LinkDef*.h'`; do
-    mkdir -p $outdir/`dirname $f`
-    cp $srcdir/$f $outdir/$f
-done
-
-cat $cppflags.tmp | sort | uniq | grep -v $srcdir | grep -v `pwd` > $cppflags
-
-echo
-echo Generating PCH for ${selmodules}
-echo