Skip to content

Improved compile speed by running multi-threaded library discovery. #2625

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Jul 25, 2025
Merged

Improved compile speed by running multi-threaded library discovery. #2625

Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
97451d8
Simplified error reporting in library detection
cmaglie May 31, 2024
9b7b9d2
Remove useless targetFilePath variable
cmaglie Jun 1, 2024
a75945a
Slight improvement of removeBuildFromSketchFiles
cmaglie Jun 2, 2024
c00b637
Rename variables for clarity
cmaglie Jun 2, 2024
ddd1e5b
Removed hardcoded build.warn_data_percentage in build.options file
cmaglie Jun 2, 2024
688f28f
Renamed variables for clarity
cmaglie Jun 2, 2024
7eb04d0
Renamed variables for clarity
cmaglie Jun 2, 2024
ba8b764
Pre-compute sourceFile fields, and save the in the includes.cache
cmaglie Jun 3, 2024
eee25d1
Added ObjFileIsUpToDate method to sourceFile
cmaglie Jun 3, 2024
6247a03
Implemented parallel task runner
cmaglie Jun 4, 2024
8013f91
Simplify use of properties.SplitQuotedString
cmaglie Jun 4, 2024
142c297
Use runner.Task in GCC preprocessor
cmaglie Jun 5, 2024
14a3adb
Parallelize library discovery phase in compile
cmaglie Jun 5, 2024
b8f94f5
The number of jobs in library detection now follows --jobs flag
cmaglie Jun 11, 2024
d0d4173
Reordered properties construction for clarity
cmaglie Jun 13, 2024
0579d02
Reordered compileFileWithRecipe for clarity
cmaglie Jun 13, 2024
b717a2a
Added integration test
cmaglie Jun 12, 2024
7241f25
fix: libraries are recompiled if the list of include paths changes
cmaglie Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Simplified error reporting in library detection 
There is no need to duplicate the preprocessResult/Err variables.
This also simplifies naming making it more straighforward.
  • Loading branch information
@cmaglie
cmaglie committed Jul 14, 2025
commit 97451d8266fb0042e78577ddba4cf624501d97a5
28 changes: 13 additions & 15 deletions internal/arduino/builder/internal/detector/detector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
}

var preprocErr error
var preprocFirstResult *runner.Result
var preprocResult *runner.Result

var missingIncludeH string
if entry := l.cache.Peek(); unchanged && entry != nil && entry.MissingIncludeH != nil {
Expand All @@ -366,20 +366,20 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
}
first = false
} else {
preprocFirstResult, preprocErr = preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)
preprocResult, preprocErr = preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)
if l.logger.VerbosityLevel() == logger.VerbosityVerbose {
l.logger.WriteStdout(preprocFirstResult.Stdout)
l.logger.WriteStdout(preprocResult.Stdout)
}
// Unwrap error and see if it is an ExitError.
var exitErr *exec.ExitError
if preprocErr == nil {
// Preprocessor successful, done
missingIncludeH = ""
} else if isExitErr := errors.As(preprocErr, &exitErr); !isExitErr || len(preprocFirstResult.Stderr) == 0 {
} else if isExitErr := errors.As(preprocErr, &exitErr); !isExitErr || len(preprocResult.Stderr) == 0 {
// Ignore ExitErrors (e.g. gcc returning non-zero status), but bail out on other errors
return preprocErr
} else {
missingIncludeH = IncludesFinderWithRegExp(string(preprocFirstResult.Stderr))
missingIncludeH = IncludesFinderWithRegExp(string(preprocResult.Stderr))
if missingIncludeH == "" && l.logger.VerbosityLevel() == logger.VerbosityVerbose {
l.logger.Info(i18n.Tr("Error while detecting libraries included by %[1]s", sourcePath))
}
Expand All @@ -396,25 +396,23 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
library := l.resolveLibrary(missingIncludeH, platformArch)
if library == nil {
// Library could not be resolved, show error
if preprocErr == nil || len(preprocFirstResult.Stderr) == 0 {
// Filename came from cache, so run preprocessor to obtain error to show
result, err := preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)

// If preprocess result came from cache, run the preprocessor to obtain the actual error to show
if preprocErr == nil || len(preprocResult.Stderr) == 0 {
preprocResult, preprocErr = preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties)
if l.logger.VerbosityLevel() == logger.VerbosityVerbose {
l.logger.WriteStdout(result.Stdout)
l.logger.WriteStdout(preprocResult.Stdout)
}
if err == nil {
if preprocErr == nil {
// If there is a missing #include in the cache, but running
// gcc does not reproduce that, there is something wrong.
// Returning an error here will cause the cache to be
// deleted, so hopefully the next compilation will succeed.
return errors.New(i18n.Tr("Internal error in cache"))
}
l.diagnosticStore.Parse(result.Args, result.Stderr)
l.logger.WriteStderr(result.Stderr)
return err
}
l.diagnosticStore.Parse(preprocFirstResult.Args, preprocFirstResult.Stderr)
l.logger.WriteStderr(preprocFirstResult.Stderr)
l.diagnosticStore.Parse(preprocResult.Args, preprocResult.Stderr)
l.logger.WriteStderr(preprocResult.Stderr)
return preprocErr
}

Expand Down