Make collection type options appendable
This PR add a new option name (e.g. --option-name+) for all list and map type options so that the data from the command line can be appended to the config file data instead of overwriting it.
MODIFY
src/main/java/de/fraunhofer/aisec/codyze/Main.java
src/main/java/de/fraunhofer/aisec/codyze/Main.java
@@ -54,7 +54,11 @@ public class Main {
cmd.parseArgs(args); // first pass to get potential config file path
if (cmd.isUsageHelpRequested()) {
// print help message
- CommandLine c = new CommandLine(new Help());
+ Help help = new Help();
+ System.setProperty("source", Arrays.toString(help.configuration.getSource()));
+ System.setProperty("mark", Arrays.toString(help.codyzeConfig.getMark()));
+
+ CommandLine c = new CommandLine(help);
c.getHelpSectionMap().put(SECTION_KEY_OPTION_LIST, new HelpRenderer());
c.usage(System.out);
System.exit(c.getCommandSpec().exitCodeOnUsageHelp());
@@ -136,7 +140,7 @@ public class Main {
@Command(mixinStandardHelpOptions = true)
public static class ConfigFilePath {
@Option(names = {
- "--config" }, paramLabel = "<path>", fallbackValue = "codyze.yaml", arity = "0..1", description = "Parse configuration settings from this file. If no file path is specified, codyze will try to load the configuration file from ./codyze.yaml")
+ "--config" }, paramLabel = "<path>", fallbackValue = "codyze.yaml", arity = "0..1", description = "Parse configuration settings from this file. If no file path is specified, codyze will try to load the configuration file from ${FALLBACK-VALUE}")
public File configFile;
@Unmatched
@@ -151,10 +155,10 @@ public class Main {
private ConfigFilePath configFilePath;
// ArgGroups only for display purposes
- @ArgGroup(heading = "@|bold,underline Codyze Options|@\n", exclusive = false)
+ @ArgGroup(heading = "@|bold,underline Codyze Options|@\n", exclusive = false, validate = true)
private CodyzeConfiguration codyzeConfig = new CodyzeConfiguration();
- @ArgGroup(heading = "", exclusive = false)
+ @ArgGroup(exclusive = false)
private Configuration configuration = new Configuration();
@ArgGroup(exclusive = false, heading = "Analysis Options\n")
@@ -182,7 +186,7 @@ public class Main {
for (CommandSpec c : mix.values()) {
sb.append(help.optionListExcludingGroups(c.options().stream().filter(optionSpec -> optionSpec.group() == null).collect(Collectors.toList())));
}
- sb.append("\n");
+
for (ArgGroupSpec group : spec.argGroups()) {
addHierachy(group, sb);
}
@@ -191,13 +195,15 @@ public class Main {
}
private void addHierachy(ArgGroupSpec argGroupSpec, StringBuilder sb) {
+ if (argGroupSpec.heading() != null) {
+ sb.append("\n");
sb.append(help.colorScheme().text(argGroupSpec.heading()).toString());
+ }
// render all options that are not in subgroups
sb.append(help.optionListExcludingGroups(
argGroupSpec.options().stream().filter(optionSpec -> optionSpec.group().equals(argGroupSpec)).collect(Collectors.toList())));
- sb.append("\n");
for (ArgGroupSpec group : argGroupSpec.subgroups()) {
addHierachy(group, sb);
}

MODIFY
src/main/java/de/fraunhofer/aisec/codyze/config/CodyzeConfiguration.kt
src/main/java/de/fraunhofer/aisec/codyze/config/CodyzeConfiguration.kt
package de.fraunhofer.aisec.codyze.config
+import com.fasterxml.jackson.annotation.JsonIgnore
import com.fasterxml.jackson.annotation.JsonProperty
import de.fraunhofer.aisec.codyze.analysis.TypestateMode
import java.io.File
+import picocli.CommandLine.ArgGroup
import picocli.CommandLine.Option
class CodyzeConfiguration {
@@ -10,12 +12,7 @@ class CodyzeConfiguration {
// TODO: names
val analysis = AnalysisMode()
- @Option(
- names = ["-m", "--mark"],
- paramLabel = "<path>",
- description = ["Loads MARK policy files.\n\t(Default: \${DEFAULT-VALUE})"],
- split = "\${sys:path.separator}"
- )
+ @JsonIgnore @ArgGroup(exclusive = true) val markCLI = MarkArgGroup()
var mark: Array<File> = arrayOf(File("./"))
// TODO: change name or make into warning levels
@@ -29,14 +26,7 @@ class CodyzeConfiguration {
)
var noGoodFindings = false
- @Option(
- names = ["--disabled-mark-rules"],
- paramLabel = "<package>.<rule>",
- description =
- [
- "The specified mark rules will be excluded from being parsed and processed. The rule has to be specified by its fully qualified name. If there is no package name, specify rule as \".<rule>\". Use \'<package>.*\' to disable an entire package."],
- split = "\${sys:path.separator}"
- )
+ @JsonIgnore @ArgGroup(exclusive = true) val disabledMarkRulesCLI = DisabledMarkArgGroup()
var disabledMarkRules: List<String> = emptyList()
@Option(
@@ -69,3 +59,65 @@ class AnalysisMode {
)
var tsMode = TypestateMode.DFA
}
+
+class MarkArgGroup {
+ var append = false
+ var matched = false
+
+ var mark: Array<File> = emptyArray()
+
+ @Option(
+ names = ["-m", "--mark"],
+ paramLabel = "<path>",
+ description = ["Loads MARK policy files.\n\t(Default: \${sys:mark})"],
+ split = "\${sys:path.separator}"
+ )
+ fun match(value: Array<File>) {
+ matched = true
+ this.mark = value
+ }
+
+ @Option(
+ names = ["--mark+"],
+ paramLabel = "<path>",
+ description =
+ ["See --mark, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: Array<File>) {
+ append = true
+ match(value)
+ }
+}
+
+class DisabledMarkArgGroup {
+ var append = false
+ var matched = false
+
+ var disabledMarkRules: List<String> = emptyList()
+ @Option(
+ names = ["--disabled-mark-rules"],
+ paramLabel = "<package>.<rule>",
+ description =
+ [
+ "The specified mark rules will be excluded from being parsed and processed. The rule has to be specified by its fully qualified name. If there is no package name, specify rule as \".<rule>\". Use \"<package>.*\" to disable an entire package."],
+ split = "\${sys:path.separator}"
+ )
+ fun match(value: List<String>) {
+ matched = true
+ this.disabledMarkRules = value
+ }
+
+ @Option(
+ names = ["--disabled-mark-rules+"],
+ paramLabel = "<package>.<rule>",
+ description =
+ [
+ "See --disabled-mark-rules, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: List<String>) {
+ append = true
+ match(value)
+ }
+}

MODIFY
src/main/java/de/fraunhofer/aisec/codyze/config/Configuration.kt
src/main/java/de/fraunhofer/aisec/codyze/config/Configuration.kt
@@ -27,29 +27,46 @@ import picocli.CommandLine
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
class Configuration {
- @JsonIgnore
- @CommandLine.ArgGroup(exclusive = true, heading = "Execution Mode\n")
- val executionMode: ExecutionMode = ExecutionMode()
-
- @CommandLine.Option(
- names = ["-s", "--source"],
- paramLabel = "<path>",
- split = "\${sys:path.separator}",
- description = ["Source files or folders to analyze.\n\t(Default: \${DEFAULT-VALUE})"]
- )
+ @JsonIgnore @CommandLine.ArgGroup(exclusive = true) private val sourceCLI = SourceArgGroup()
var source: Array<File> = arrayOf(File("./"))
private set
+ get() {
+ val result = mutableListOf<File>()
+ if (!sourceCLI.matched || sourceCLI.append) {
+ result.addAll(field)
+ }
+
+ if (sourceCLI.matched) {
+ result.addAll(sourceCLI.source)
+ }
+
+ return result.toTypedArray()
+ }
+
+ @JsonIgnore
+ @CommandLine.ArgGroup(exclusive = true)
+ private val disabledSourceCLI = DisabledSourceArgGroup()
@JsonProperty("disabled-sources")
- @CommandLine.Option(
- names = ["--disabled-sources"],
- paramLabel = "<path>",
- split = "\${sys:path.separator}",
- description =
- [
- "Files or folders specified here will not be analyzed. Symbolic links are not followed when filtering out these paths"]
- )
var disabledSource: Array<File> = emptyArray()
+ private set
+ get() {
+ val result = mutableListOf<File>()
+
+ if (!disabledSourceCLI.matched || disabledSourceCLI.append) {
+ result.addAll(field)
+ }
+
+ if (disabledSourceCLI.matched) {
+ result.addAll(disabledSourceCLI.disabledSource)
+ }
+
+ return result.toTypedArray()
+ }
+
+ @JsonIgnore
+ @CommandLine.ArgGroup(exclusive = true, heading = "Execution Mode\n")
+ val executionMode: ExecutionMode = ExecutionMode()
// TODO output standard stdout?
@CommandLine.Option(
@@ -104,11 +121,19 @@ class Configuration {
.launchConsole(executionMode.isTui)
.typestateAnalysis(codyze.analysis.tsMode)
.disableGoodFindings(codyze.noGoodFindings)
- .markFiles(*codyze.mark.map { m -> m.absolutePath }.toTypedArray())
.pedantic(codyze.pedantic)
+ val mark = mutableListOf<File>()
+ if (!codyze.markCLI.matched || codyze.markCLI.append) mark.addAll(codyze.mark)
+ if (codyze.markCLI.matched) mark.addAll(codyze.markCLI.mark)
+ config.markFiles(*mark.map { m -> m.absolutePath }.toTypedArray())
+
val disabledRulesMap = mutableMapOf<String, DisabledMarkRulesValue>()
- for (mName in codyze.disabledMarkRules) {
+ val disabledMarkRules = codyze.disabledMarkRulesCLI.disabledMarkRules.toMutableList()
+ if (!codyze.disabledMarkRulesCLI.matched || codyze.disabledMarkRulesCLI.append)
+ disabledMarkRules.addAll(codyze.disabledMarkRules)
+
+ for (mName in disabledMarkRules) {
val index = mName.lastIndexOf('.')
val packageName = mName.subSequence(0, index).toString()
val markName = mName.subSequence(index + 1, mName.length).toString()
@@ -120,9 +145,10 @@ class Configuration {
log.warn(
"Error while parsing disabled-mark-rules: \'$mName\' is not a valid name for a mark rule. Continue parsing disabled-mark-rules"
)
- config.disableMark(disabledRulesMap)
}
+ config.disableMark(disabledRulesMap)
+
return config.build()
}
@@ -141,42 +167,77 @@ class Configuration {
.loadIncludes(cpg.translation.analyzeIncludes)
.useUnityBuild(cpg.useUnityBuild)
.processAnnotations(cpg.processAnnotations)
- .symbols(cpg.symbols)
.useParallelFrontends(cpg.useParallelFrontends)
.typeSystemActiveInFrontend(cpg.typeSystemInFrontend)
.defaultLanguages()
.sourceLocations(*filterFiles(disabledSource, *sources))
+ val symbols = cpg.symbolsCLI.symbols.toMutableMap()
+ if (!cpg.symbolsCLI.matched || cpg.symbolsCLI.append) {
+ symbols.putAll(cpg.symbols)
+ }
+ translationConfig.symbols(symbols)
+
+ if (!cpg.translation.includesCLI.matched || cpg.translation.includesCLI.append) {
for (file in cpg.translation.includes) {
translationConfig.includePath(file.absolutePath)
}
+ }
+ if (cpg.translation.includesCLI.matched)
+ for (file in cpg.translation.includesCLI.includes) {
+ translationConfig.includePath(file.absolutePath)
+ }
+
+ if (!cpg.translation.enabledIncludesCLI.matched || cpg.translation.enabledIncludesCLI.append
+ ) {
for (s in cpg.translation.enabledIncludes) {
translationConfig.includeWhitelist(s.absolutePath)
}
+ }
+ if (cpg.translation.enabledIncludesCLI.matched)
+ for (file in cpg.translation.enabledIncludesCLI.enabledIncludes) {
+ translationConfig.includeWhitelist(file.absolutePath)
+ }
+
+ if (!cpg.translation.disabledIncludesCLI.matched ||
+ cpg.translation.disabledIncludesCLI.append
+ ) {
for (s in cpg.translation.disabledIncludes) {
translationConfig.includeBlacklist(s.absolutePath)
}
+ }
+ if (cpg.translation.disabledIncludesCLI.matched)
+ for (file in cpg.translation.disabledIncludesCLI.disabledIncludes) {
+ translationConfig.includeBlacklist(file.absolutePath)
+ }
if (cpg.disableCleanup) {
translationConfig.disableCleanup()
}
if (cpg.defaultPasses == null) {
- if (cpg.passes.isEmpty()) {
+ if (cpg.passesCLI.passes.isEmpty() && cpg.passes.isEmpty()) {
translationConfig.defaultPasses()
}
} else {
if (cpg.defaultPasses!!) {
translationConfig.defaultPasses()
} else {
- if (cpg.passes.isEmpty()) {
+ if (cpg.passesCLI.passes.isEmpty() && cpg.passes.isEmpty()) {
// TODO: error handling for no passes if needed
}
}
}
+
+ if (!cpg.passesCLI.matched || cpg.passesCLI.append) {
for (p in cpg.passes) {
translationConfig.registerPass(p)
}
+ }
+ if (cpg.passesCLI.matched)
+ for (p in cpg.passesCLI.passes) {
+ translationConfig.registerPass(p)
+ }
for (l in cpg.additionalLanguages) {
val frontendClazz =
@@ -277,6 +338,7 @@ class Configuration {
if (codyze.pedantic) {
codyze.noGoodFindings = false
codyze.disabledMarkRules = emptyList()
+ codyze.disabledMarkRulesCLI.disabledMarkRules = emptyList()
}
// we need to force load includes for unity builds, otherwise nothing will be parsed
@@ -440,6 +502,67 @@ class ExecutionMode {
var isTui = false
}
+class SourceArgGroup {
+ var append = false
+ var matched = false
+
+ var source: Array<File> = emptyArray()
+ @CommandLine.Option(
+ names = ["-s", "--source"],
+ paramLabel = "<path>",
+ split = "\${sys:path.separator}",
+ description = ["Source files or folders to analyze.\n\t(Default: \${sys:source})"]
+ )
+ fun match(value: Array<File>) {
+ matched = true
+ this.source = value
+ }
+
+ @CommandLine.Option(
+ names = ["--source+"],
+ paramLabel = "<path>",
+ split = "\${sys:path.separator}",
+ description =
+ ["See --source, but appends the values to the ones specified in configuration file."]
+ )
+ fun append(value: Array<File>) {
+ append = true
+ match(value)
+ }
+}
+
+class DisabledSourceArgGroup {
+ var append = false
+ var matched = false
+
+ var disabledSource: Array<File> = emptyArray()
+ @CommandLine.Option(
+ names = ["--disabled-sources"],
+ paramLabel = "<path>",
+ split = "\${sys:path.separator}",
+ description =
+ [
+ "Files or folders specified here will not be analyzed. Symbolic links are not followed when filtering out these paths"]
+ )
+ fun match(value: Array<File>) {
+ matched = true
+ this.disabledSource = value
+ }
+
+ @CommandLine.Option(
+ names = ["--disabled-sources+"],
+ paramLabel = "<path>",
+ split = "\${sys:path.separator}",
+ description =
+ [
+ "See --disabled-sources, but appends the values to the ones specified in configuration file."]
+ )
+ fun append(value: Array<File>) {
+ append = true
+ match(value)
+ }
+}
+
// Taken from:
// https://stackoverflow.com/questions/46644099/cant-set-problemhandler-to-objectmapper-in-spring-boot
class LogModule(private val problemHandler: LogDeserializationProblemHandler) : SimpleModule() {

MODIFY
src/main/java/de/fraunhofer/aisec/codyze/config/CpgConfiguration.kt
src/main/java/de/fraunhofer/aisec/codyze/config/CpgConfiguration.kt
package de.fraunhofer.aisec.codyze.config
+import com.fasterxml.jackson.annotation.JsonIgnore
import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
import de.fraunhofer.aisec.codyze.config.converters.LanguageDeserializer
@@ -7,6 +8,7 @@ import de.fraunhofer.aisec.codyze.config.converters.PassListDeserializer
import de.fraunhofer.aisec.cpg.passes.Pass
import java.io.File
import java.util.*
+import picocli.CommandLine.ArgGroup
import picocli.CommandLine.Option
/** This class consists of configuration settings for CPG. */
@@ -44,29 +46,6 @@ class CpgConfiguration {
)
var typeSystemInFrontend = true
- @Option(
- names = ["--default-passes"],
- negatable = true,
- description = ["Controls the usage of default passes for cpg.\n\t(Default: true)"],
- fallbackValue = "true"
- )
- // Set to null to differentiate if it was set or not
- var defaultPasses: Boolean? = null
-
- @Option(
- names = ["--passes"],
- paramLabel = "<pass>",
- description =
- [
- "CPG passes in the order in which they should be executed, fully qualified name of the classes only. If default-passes is specified, the default passes are executed first."],
- split = "\${sys:path.separator}"
- )
- @JvmName("setPassesNull")
- fun setPasses(passes: List<Pass?>) {
- this.passes = passes.filterNotNull()
- }
- @JsonDeserialize(using = PassListDeserializer::class) var passes: List<Pass> = ArrayList()
-
@Option(
names = ["--debug-parser"],
description = ["Controls debug output generation for the cpg parser."],
@@ -110,12 +89,7 @@ class CpgConfiguration {
)
var failOnError = false
- @Option(
- names = ["--symbols"],
- paramLabel = "<symbol>=<definition>",
- description = ["Definition of additional symbols."],
- split = "\${sys:path.separator}"
- )
+ @JsonIgnore @ArgGroup(exclusive = true) val symbolsCLI = SymbolsArgGroup()
var symbols: Map<String, String> = HashMap()
@JsonProperty("parallel-frontends")
@@ -127,6 +101,18 @@ class CpgConfiguration {
fallbackValue = "true"
)
var useParallelFrontends = false
+
+ @Option(
+ names = ["--default-passes"],
+ negatable = true,
+ description = ["Controls the usage of default passes for cpg.\n\t(Default: true)"],
+ fallbackValue = "true"
+ )
+ // Set to null to differentiate if it was set or not
+ var defaultPasses: Boolean? = null
+
+ @JsonIgnore @ArgGroup(exclusive = true) val passesCLI = PassesArgGroup()
+ @JsonDeserialize(using = PassListDeserializer::class) var passes: List<Pass> = ArrayList()
}
/** This class consists of settings that control how CPG will process includes. */
@@ -140,13 +126,113 @@ class TranslationSettings {
)
var analyzeIncludes = false
+ @JsonIgnore @ArgGroup(exclusive = true) val includesCLI = IncludesArgGroup()
+ var includes: Array<File> = emptyArray()
+
+ @JsonIgnore @ArgGroup(exclusive = true) val enabledIncludesCLI = EnabledIncludesArgGroup()
+ var enabledIncludes: Array<File> = emptyArray()
+
+ @JsonIgnore @ArgGroup(exclusive = true) val disabledIncludesCLI = DisabledIncludesArgGroup()
+ var disabledIncludes: Array<File> = emptyArray()
+}
+
+class PassesArgGroup {
+ var append = false
+ var matched = false
+
+ var passes: List<Pass> = emptyList()
+
+ @Option(
+ names = ["--passes"],
+ paramLabel = "<pass>",
+ description =
+ [
+ "CPG passes in the order in which they should be executed, fully qualified name of the classes only. If default-passes is specified, the default passes are executed first."],
+ split = "\${sys:path.separator}"
+ )
+ fun match(passes: List<Pass?>) {
+ matched = true
+ this.passes = passes.filterNotNull()
+ }
+
+ @Option(
+ names = ["--passes+"],
+ paramLabel = "<pass>",
+ description =
+ ["See passes, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: List<Pass?>) {
+ append = true
+ match(value)
+ }
+}
+
+class SymbolsArgGroup {
+ var append = false
+ var matched = false
+
+ var symbols: Map<String, String> = emptyMap()
+
+ @Option(
+ names = ["--symbols"],
+ paramLabel = "<symbol>=<definition>",
+ description = ["Definition of additional symbols."],
+ split = "\${sys:path.separator}"
+ )
+ fun match(value: Map<String, String>) {
+ matched = true
+ this.symbols = value
+ }
+
+ @Option(
+ names = ["--symbols+"],
+ paramLabel = "<symbol>=<definition>",
+ description =
+ ["See --symbols, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: Map<String, String>) {
+ append = true
+ match(value)
+ }
+}
+
+class IncludesArgGroup {
+ var append = false
+ var matched = false
+
+ var includes: Array<File> = emptyArray()
+
@Option(
names = ["--includes"],
paramLabel = "<path>",
description = ["Path(s) containing include files."],
split = "\${sys:path.separator}"
)
- var includes: Array<File> = emptyArray()
+ fun match(value: Array<File>) {
+ matched = true
+ this.includes = value
+ }
+
+ @Option(
+ names = ["--includes+"],
+ paramLabel = "<path>",
+ description =
+ ["See --includes, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: Array<File>) {
+ append = true
+ match(value)
+ }
+}
+
+class EnabledIncludesArgGroup {
+ var append = false
+ var matched = false
+
+ var enabledIncludes: Array<File> = emptyArray()
@Option(
names = ["--enabled-includes"],
@@ -156,7 +242,30 @@ class TranslationSettings {
"If includes is not empty, only the specified files will be parsed and processed in the cpg, unless it is a part of the disabled list, in which it will be ignored."],
split = "\${sys:path.separator}"
)
- var enabledIncludes: Array<File> = emptyArray()
+ fun match(value: Array<File>) {
+ matched = true
+ this.enabledIncludes = value
+ }
+
+ @Option(
+ names = ["--enabled-includes+"],
+ paramLabel = "<path>",
+ description =
+ [
+ "See --enabled-includes, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: Array<File>) {
+ append = true
+ match(value)
+ }
+}
+
+class DisabledIncludesArgGroup {
+ var append = false
+ var matched = false
+
+ var disabledIncludes: Array<File> = emptyArray()
@Option(
names = ["--disabled-includes"],
@@ -166,5 +275,21 @@ class TranslationSettings {
"If includes is not empty, the specified files will be excluded from being parsed and processed in the cpg. The disabled list entries always take priority over the enabled list entries."],
split = "\${sys:path.separator}"
)
- var disabledIncludes: Array<File> = emptyArray()
+ fun match(value: Array<File>) {
+ matched = true
+ this.disabledIncludes = value
+ }
+
+ @Option(
+ names = ["--disabled-includes+"],
+ paramLabel = "<path>",
+ description =
+ [
+ "See --disabled-includes, but appends the values to the ones specified in configuration file."],
+ split = "\${sys:path.separator}"
+ )
+ fun append(value: Array<File>) {
+ append = true
+ match(value)
+ }
}

MODIFY
src/test/java/de/fraunhofer/aisec/codyze/crymlin/ConfigCLILoadTest.kt
src/test/java/de/fraunhofer/aisec/codyze/crymlin/ConfigCLILoadTest.kt
@@ -2,9 +2,12 @@ package de.fraunhofer.aisec.codyze.crymlin
import de.fraunhofer.aisec.codyze.analysis.TypestateMode
import de.fraunhofer.aisec.codyze.config.Configuration
+import de.fraunhofer.aisec.codyze.config.DisabledMarkRulesValue
import de.fraunhofer.aisec.cpg.TranslationConfiguration
import de.fraunhofer.aisec.cpg.passes.CallResolver
+import de.fraunhofer.aisec.cpg.passes.EdgeCachePass
import de.fraunhofer.aisec.cpg.passes.FilenameMapper
+import de.fraunhofer.aisec.cpg.passes.UnreachableEOGPass
import java.io.File
import java.lang.Exception
import kotlin.Throws
@@ -144,10 +147,247 @@ class ConfigCLILoadTest {
assertContentEquals(expectedDisabledIncludes, translationConfiguration.includeBlacklist)
}
+ @Test
+ @Throws(Exception::class)
+ fun appendTest() {
+ val options =
+ arrayOf(
+ "--mark+=mark5${File.pathSeparator}mark7${File.pathSeparator}mark6",
+ "--includes+=include7${File.pathSeparator}include193${File.pathSeparator}include3",
+ "--source+=cliSource.java${File.pathSeparator}cliDir"
+ )
+ val config = Configuration.initConfig(correctFile, *options)
+ val serverConfig = config.buildServerConfiguration()
+ val translationConfig = config.buildTranslationConfiguration()
+ val configFileBasePath = correctFile.absoluteFile.parent
+
+ val expectedConfigMark =
+ listOf("mark1", "mark4", "mark3", "mark2")
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedConfigMark, "mark5", "mark7", "mark6")
+ .map { s -> File(s).absolutePath }
+ .toTypedArray(),
+ serverConfig.markModelFiles,
+ "Option specified in config file should be appended to CLI option"
+ )
+
+ val expectedConfigIncludes =
+ listOf("include1", "include2")
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedConfigIncludes, "include7", "include193", "include3")
+ .map { s -> File(s).absolutePath }
+ .toTypedArray(),
+ translationConfig.includePaths,
+ "Option specified in config file should be appended to CLI option"
+ )
+
+ assertContentEquals(
+ arrayOf(
+ File(configFileBasePath, "source.java"),
+ File("cliSource.java"),
+ File("cliDir")
+ ),
+ config.source,
+ "Option specified in config file should be appended to CLI option"
+ )
+ }
+
+ @Test
+ @Throws(java.lang.Exception::class)
+ fun additionalAppendTest() {
+ val config =
+ Configuration.initConfig(
+ additionalOptionFile,
+ "-c",
+ "--passes+=de.fraunhofer.aisec.cpg.passes.FilenameMapper${File.pathSeparator}" +
+ "de.fraunhofer.aisec.cpg.passes.CallResolver",
+ "--symbols+=&=and${File.pathSeparator}+=plus",
+ "--default-passes",
+ "--includes+=include9${File.pathSeparator}include193${File.pathSeparator}include13",
+ "--enabled-includes+=include9${File.pathSeparator}include52",
+ "--disabled-includes+=include13"
+ )
+ val translationConfiguration = config.buildTranslationConfiguration(File("test.java"))
+ val expectedTranslationConfig =
+ TranslationConfiguration.builder()
+ .defaultPasses()
+ .registerPass(EdgeCachePass())
+ .registerPass(UnreachableEOGPass())
+ .registerPass(FilenameMapper())
+ .registerPass(CallResolver())
+ .build()
+
+ val configFileBasePath = additionalOptionFile.absoluteFile.parent
+
+ assertEquals(
+ expectedTranslationConfig.registeredPasses.size,
+ translationConfiguration.registeredPasses.size,
+ "Expected size ${expectedTranslationConfig.registeredPasses.size} but was ${translationConfiguration.registeredPasses.size}"
+ )
+ val passesNames =
+ translationConfiguration.registeredPasses.map { s -> s.javaClass.name }.toTypedArray()
+ val expectedPassesNames =
+ expectedTranslationConfig.registeredPasses.map { s -> s.javaClass.name }.toTypedArray()
+ assertContentEquals(expectedPassesNames, passesNames)
+
+ assertEquals(
+ 5,
+ translationConfiguration.symbols.size,
+ "Expected size 5 but was ${translationConfiguration.symbols.size}"
+ )
+ assertTrue(
+ translationConfiguration.symbols.containsKey("#"),
+ "Did not contain \'#\' as a key"
+ )
+ assertEquals("hash", translationConfiguration.symbols["#"])
+ assertTrue(
+ translationConfiguration.symbols.containsKey("$"),
+ "Did not contain \'$\' as a key"
+ )
+ assertEquals("dollar", translationConfiguration.symbols["$"])
+ assertTrue(
+ translationConfiguration.symbols.containsKey("*"),
+ "Did not contain \'*\' as a key"
+ )
+ assertEquals("star", translationConfiguration.symbols["*"])
+ assertTrue(
+ translationConfiguration.symbols.containsKey("&"),
+ "Did not contain \'&\' as a key"
+ )
+ assertEquals("and", translationConfiguration.symbols["&"])
+ assertTrue(
+ translationConfiguration.symbols.containsKey("+"),
+ "Did not contain \'=\' as a key"
+ )
+ assertEquals("plus", translationConfiguration.symbols["+"])
+
+ val expectedIncludes =
+ arrayOf("include1", "include7", "include3", "include5")
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedIncludes, "include9", "include193", "include13")
+ .map { s -> File(s).absolutePath }
+ .toTypedArray(),
+ translationConfiguration.includePaths
+ )
+
+ val expectedEnabledIncludes =
+ arrayOf("include3", "include5", "include1")
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedEnabledIncludes, "include9", "include52").map { s ->
+ File(s).absolutePath
+ },
+ translationConfiguration.includeWhitelist
+ )
+
+ val expectedDisabledIncludes =
+ arrayOf("include7", "include3")
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedDisabledIncludes, "include13").map { s -> File(s).absolutePath },
+ translationConfiguration.includeBlacklist
+ )
+ }
+
+ @Test
+ fun disabledMarkAppendTest() {
+ val config =
+ Configuration.initConfig(
+ disabledMarkFile,
+ "--disabled-mark-rules+=package.mark0${File.pathSeparator}package2.*"
+ )
+ val serverConfiguration = config.buildServerConfiguration()
+ val expectedMap =
+ mapOf(
+ Pair("package", DisabledMarkRulesValue(false, mutableSetOf("mark1", "mark0"))),
+ Pair("", DisabledMarkRulesValue(false, mutableSetOf("mark2"))),
+ Pair("package0", DisabledMarkRulesValue(false, mutableSetOf("mark123"))),
+ Pair("package2", DisabledMarkRulesValue(true))
+ )
+
+ for (expectedKey in expectedMap.keys) {
+ assertContains(
+ serverConfiguration.packageToDisabledMarkRules,
+ expectedKey,
+ "\"$expectedKey\" was not a key in map"
+ )
+
+ val expectedValue = expectedMap[expectedKey]
+ val actualValue = serverConfiguration.packageToDisabledMarkRules[expectedKey]
+ if (expectedValue != null && actualValue != null) {
+ assertEquals(
+ expectedValue.isDisablePackage,
+ actualValue.isDisablePackage,
+ "Boolean to disable entire package was not equal"
+ )
+ for (rule in expectedValue.disabledMarkRuleNames) {
+ assertTrue(
+ actualValue.disabledMarkRuleNames.contains(rule),
+ "Rule $rule was not in set"
+ )
+ }
+ }
+ }
+ }
+
+ @Test
+ fun disabledSourceAppendTest() {
+ val options =
+ arrayOf(
+ "--source+=cliDir1${File.pathSeparator}cliDir2/cliSource.java${File.pathSeparator}cliSource.java",
+ "--disabled-sources+=cliDir2${File.pathSeparator}cliSource.java"
+ )
+ val config = Configuration.initConfig(sourceDisablingFile, *options)
+ val configFileBasePath = sourceDisablingFile.absoluteFile.parent
+
+ val expectedConfigSource =
+ arrayOf(
+ "../real-examples/botan/blockciphers/Antidote1911.Arsenic",
+ "../real-examples/botan/blockciphers/obraunsdorf.playbook-creator",
+ "../real-examples/botan/MARK",
+ "../real-examples/botan/MARK",
+ "../real-examples/botan/streamciphers",
+ "../directory-structure"
+ )
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedConfigSource, "cliDir1", "cliDir2/cliSource.java", "cliSource.java")
+ .map { s -> File(s).absolutePath },
+ config.source.map { f -> f.absolutePath }
+ )
+
+ val expectedConfigDisabledSource =
+ arrayOf(
+ "../real-examples/botan/blockciphers",
+ "../config-files/additional_options.yml",
+ "../real-examples/botan/MARK",
+ "../directory-structure/dir2/dir2dir1/dir2dir1file1.java"
+ )
+ .map { s -> File(configFileBasePath, s).absolutePath }
+ .toTypedArray()
+ assertContentEquals(
+ arrayOf(*expectedConfigDisabledSource, "cliDir2", "cliSource.java").map { s ->
+ File(s).absolutePath
+ },
+ config.disabledSource.map { f -> f.absolutePath }
+ )
+ }
+
companion object {
private lateinit var correctFile: File
private lateinit var incorrectFile: File
private lateinit var additionalOptionFile: File
+ private lateinit var disabledMarkFile: File
+ private lateinit var sourceDisablingFile: File
@BeforeAll
@JvmStatic
@@ -175,6 +415,20 @@ class ConfigCLILoadTest {
assertNotNull(additionalOptionResource)
additionalOptionFile = File(additionalOptionResource.file)
assertNotNull(additionalOptionFile)
+
+ val disabledMarkResource =
+ ConfigLoadTest::class.java.classLoader.getResource("config-files/disabled_mark.yml")
+ assertNotNull(disabledMarkResource)
+ disabledMarkFile = File(disabledMarkResource.file)
+ assertNotNull(disabledMarkFile)
+
+ val sourceDisablingResource =
+ ConfigLoadTest::class.java.classLoader.getResource(
+ "config-files/source_disabling.yml"
+ )
+ assertNotNull(sourceDisablingResource)
+ sourceDisablingFile = File(sourceDisablingResource.file)
+ assertNotNull(sourceDisablingFile)
}
}
}

ADD
src/test/resources/config-files/disabled_mark.yml
+codyze:
+ disabled-mark-rules:
+ - package.mark1
+ - .mark2
+ - package0.mark123
\ No newline at end of file

Make collection type options appendable (#463)

This PR add a new option name (e.g. --option-name+) for all list and map type options so that the data from the command line can be appended to the config file data instead of overwriting it.

Co-authored-by: Florian Wendland <florian.wendland@aisec.fraunhofer.de>