scons dependency problems in a hierarchical build with variant dirs - scons

Question

Im new to scons and am having problems with scons dependancies
in a hierarchichal build with a variant directory.
Im able to reproduce the problem in a reduced environment that consists of
2 subdirs under the SConscript directory (moduleA and moduleB) as follows:
.
|-- SConstruct
|-- file.conf
|-- moduleA
| |-- SConscript
| `-- conf2cc
`-- moduleB
|-- SConscript
`-- fileB.cc
Here is the flow of what needs to be done:
moduleA executes a shell script: conf2cc, input: $projRootDir/file.conf, output: moduleA/$variantDir/source.cc
moduleA compiles source.cc and creates moduleA/$variantDir/libmoduleA.a
moduleB needs to copy moduleA/$variantDir/source.cc to moduleB/source.cc
moduleB needs to compile moduleB/source.cc and moduleB/fileB.cc into its
library libmoduleB.a
Its entirely possible that Im doing several things wrong here. For example, I know
Im not using $TARGET/$SOURCE in moduleA Command(), but thats on purpose, since the
script needs the absolute path names, and scons doesnt remove the leading '#'
The problem I have is the Command() builder in moduleB (step 3 above) never executes.
Here are the SConstruct and SConscript files:
Sconstruct
import os
env = Environment()
env['variantDir'] = 'linux' # for this example, just make variantDir = linux
modules = ['moduleA', 'moduleB']
for dir in modules:
SConscript(
os.path.join(dir, 'SConscript'),
variant_dir = os.path.join(dir, env['variantDir']),
exports = ['env'],
duplicate = 0)
moduleA/Sconscript
import os
Import('env')
scriptInput = '#file.conf'
sourceFile = os.path.join('#moduleA', env['variantDir'], 'source.cc')
conf2ccScript = File('#moduleA/conf2cc').abspath
# The script needs abspaths for input and output, not the scons '#' prepended
# the script syntax is: script <inputFile> <outputFile>
cmd = '%s %s %s' % (conf2ccScript, File(scriptInput).abspath, File(sourceFile).abspath)
# Generate source.cc file based on file.conf
conf2ccNode = env.Command(target = sourceFile,
source = scriptInput,
action = cmd)
libNode = env.Library(target = 'moduleA', source = sourceFile)
env.Depends(libNode, conf2ccNode)
moduleB/Sconscript
import os
Import('env')
sourceFiles = ['fileB.cc', 'source.cc']
# Get the source.cc file
externalSourceFile = os.path.join('#moduleA', env['variantDir'], 'source.cc')
sourceTarget = os.path.join('#moduleB', 'source.cc')
cmdNode = env.Command(target = sourceTarget,
source = externalSourceFile,
action = Copy('$TARGET', '$SOURCE'))
libNode = env.Library(target = 'moduleB', source = sourceFiles)
env.Depends(libNode, cmdNode)
Here is the output when I execute scons:
Any help would be greatly appreciated!
Brady
notroot#ubuntu:~/projects/sconsTest/sconsTestHierDeps$ scons
scons: Reading SConscript files ...
scons: done reading SConscript files.
scons: Building targets ...
/home/notroot/projects/sconsTest/sconsTestHierDeps/moduleA/conf2cc /home/notroot/projects/sconsTest/sconsTestHierDeps/file.conf /home/notroot/projects/sconsTest/sconsTestHierDeps/moduleA/linux/source.cc
g++ -o moduleA/linux/source.o -c moduleA/linux/source.cc
ar rc moduleA/linux/libmoduleA.a moduleA/linux/source.o
ranlib moduleA/linux/libmoduleA.a
g++ -o moduleB/linux/fileB.o -c moduleB/fileB.cc
scons: *** [moduleB/linux/source.o] Source `moduleB/source.cc' not found, needed by target `moduleB/linux/source.o'.
scons: building terminated because of errors.

Answer 1

I suggest problem in incorrect dependency what you use and filenames.
May be problem in variant_dir and source files for moduleB, you use Command to generate #/moduleB/source.cc, but in sourceFiles you have 'source.cc'.
So, one of ways to help you may be correct moduleB SConscript :
# Get the source.cc file
externalSourceFile = os.path.join('#moduleA', env['variantDir'], 'source.cc')
sourceTarget = os.path.join('#moduleB', 'source.cc')
sourceFiles = ['fileB.cc', sourceTarget]
cmdNode = env.Command(target = sourceTarget,
source = externalSourceFile,
action = Copy('$TARGET', '$SOURCE'))
libNode = env.Library(target = 'moduleB', source = sourceFiles)
And try to use Command like source file. instead filename. It's looks more correct.
moduleA :
conf2ccNode = env.Command(target = sourceFile,
source = scriptInput,
action = cmd)
libNode = env.Library(target = 'moduleA', source = conf2ccNode)
moduleB:
cmdNode = env.Command(target = sourceTarget,
source = externalSourceFile,
action = Copy('$TARGET', '$SOURCE'))
libNode = env.Library(target = 'moduleB', source = ['fileB.cc', cmdNode])

Answer 2

I found a solution to the problem, but I dont really understand why it works.
If I add a call to env.Default() with the targets I need built, then it works. So the SConscript files would then look like this:
moduleA/Sconscript
import os
Import('env')
scriptInput = '#file.conf'
sourceFile = os.path.join('#moduleA', env['variantDir'], 'source.cc')
conf2ccScript = File('#moduleA/conf2cc').abspath
# The script needs abspaths for input and output, not the scons '#' prepended
# the script syntax is: script <inputFile> <outputFile>
cmd = '%s %s %s' % (conf2ccScript, File(scriptInput).abspath, File(sourceFile).abspath)
# Generate source.cc file based on file.conf
conf2ccNode = env.Command(target = sourceFile,
source = scriptInput,
action = cmd)
libNode = env.Library(target = 'moduleA', source = sourceFile)
env.Depends(libNode, conf2ccNode)
env.Default([conf2ccNode, libNode])
moduleB/Sconscript
import os
Import('env')
sourceFiles = ['fileB.cc', 'source.cc']
# Get the source.cc file
externalSourceFile = os.path.join('#moduleA', env['variantDir'], 'source.cc')
sourceTarget = os.path.join('#moduleB', 'source.cc')
cmdNode = env.Command(target = sourceTarget,
source = externalSourceFile,
action = Copy('$TARGET', '$SOURCE'))
libNode = env.Library(target = 'moduleB', source = sourceFiles)
env.Depends(libNode, cmdNode)
env.Default(cmdNode, libNode)
So that leads to the question: If I dont specify Default() targets and there is more than one target, how does scons know which one to build?
Also, I still dont understand why scons doesnt resolve the dependancy in moduleB/SConscript that libNode has on the cmdNode.

Answer 3

My solution:
def CreateLibrary(env, name, sources, shared=True):
def GetObjectFile(sourceFileName):
def GetFileNameWithoutExtension(path):
return os.path.splitext(os.path.basename(path))[0]
def IsFileNameExist(newFileName):
return fileNames.count(newFileName) > 0
sourceAbsPath = os.path.abspath(sourceFileName)
fileNameWithoutExtension = GetFileNameWithoutExtension(sourceAbsPath)
destFileName = fileNameWithoutExtension
attemptNumber = 0
while IsFileNameExist(destFileName):
attemptNumber += 1
destFileName = fileNameWithoutExtension + str(attemptNumber)
fileNames.append(destFileName)
destFilePath = os.path.join(compilationDirRoot, destFileName)
if shared:
return env.SharedObject(destFilePath, sourceAbsPath)
else:
return env.StaticObject(destFilePath, sourceAbsPath)
objFiles = []
fileNames = []
compilationDirRoot = Dir('.').abspath
for src in sources:
if isinstance(src,str):
objFiles.append(GetObjectFile(src))
elif isinstance(src, SCons.Node.FS.File):
objFiles.append(GetObjectFile(SCons.Node.FS.File.rstr(src)))
else:
for f in src:
objFiles.append(GetObjectFile(str(f)))
if shared:
return env.SharedLibrary(name, objFiles, no_import_lib=True)
else:
return env.StaticLibrary(name, objFiles)
Example of use:
theora = CreateLibrary(env, 'theora', sources)