Support for the TF `concat` function
In addition, function handling has been reworked to be more concise and allow easier addition of new functions.
MODIFY
checkov/terraform/parser.py
checkov/terraform/parser.py
+import copy
import json
import logging
import os
import re
from pathlib import Path
-from typing import Mapping, Optional, Dict, Any, List, Callable, Set, Tuple
-import copy
+from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Tuple
+
import deep_merge
import hcl2
import jmespath
from checkov.common.runners.base_runner import filter_ignored_directories
from checkov.common.util.consts import DEFAULT_EXTERNAL_MODULES_DIR, RESOLVED_MODULE_ENTRY_NAME
-from checkov.common.util.type_forcers import convert_str_to_bool
from checkov.common.variables.context import EvaluationContext, VarReference
-from checkov.terraform.module_loading.registry import ModuleLoaderRegistry
-from checkov.terraform.module_loading.registry import module_loader_registry as default_ml_registry
-from checkov.terraform.parser_var_blocks import find_var_blocks, split_merge_args
+from checkov.terraform import parser_functions
+from checkov.terraform.module_loading.registry import module_loader_registry as default_ml_registry, \
+ ModuleLoaderRegistry
+from checkov.terraform.parser_functions import FUNCTION_FAILED
+from checkov.terraform.parser_utils import eval_string, find_var_blocks, to_string
LOGGER = logging.getLogger(__name__)
@@ -366,62 +368,15 @@ class Parser:
if match.full_str == altered_value:
altered_value = replaced
else:
- altered_value = altered_value.replace(match.full_str, str(replaced))
+ replace_str = f"'{match.full_str}'"
+ if isinstance(replaced, str) or replace_str not in altered_value:
+ replace_str = match.full_str
+ altered_value = altered_value.replace(replace_str, str(replaced))
prev_matches.append((match.full_str, replaced))
had_var_block_match = True
if not had_var_block_match:
- # tomap is annoying because the curly braces in the string break the regex. Rather than
- # coming up with something that's significantly more complex, we'll special case this
- # check. Only do this when there wasn't a pattern match to make sure there are no
- # variables lingering in the map value.
- # https://www.terraform.io/docs/configuration/functions/tomap.html
- if value.startswith("${tomap(") and value.endswith(")}"):
- trimmed = value[8:-2]
- trimmed = trimmed.replace(":", "=") # converted to colons by parser #shrug
- altered_value = _eval_string(trimmed)
- if altered_value is None:
- altered_value = value
- continue
-
- if not isinstance(altered_value, dict):
- continue
-
- # If there is a string and anything else, convert to string
- had_string = False
- had_something_else = False
- for k, v in altered_value.items():
- if v == "${True}":
- altered_value[k] = True
- v = True
- elif v == "${False}":
- altered_value[k] = False
- v = False
-
- if isinstance(v, str):
- had_string = True
- if had_something_else:
- break
- else:
- had_something_else = True
- if had_string:
- break
- if had_string and had_something_else:
- altered_value = {k: _tostring(v) for k, v in altered_value.items()}
- # Same as above, regex can blow this up
- # (see parser scenario: tostring_function, INNER_CURLY
- elif value.startswith("${tostring(\"") and value.endswith("\")}"):
- altered_value = value[12:-3]
-
- # Support HCL 0.11 optional boolean syntax - evaluate "true" to true and "false" to false
- #
- # `allow_str_bool_translation` exists because we want to prevent conversion in a dict
- # which is a direct value. See the "MIXED_BOOL" variable in the "tomap_function" parser
- # scenario for a situation which worked incorrectly without this.
- # NOTE: This is probably not a big deal to be removed if this causes problems in other
- # places. The MIXED_BOOL test case is technically correct with the TF spec, but
- # isn't essential operation for Checkov.
- elif allow_str_bool_translation and value == "true":
+ if allow_str_bool_translation and value == "true":
altered_value = True
elif allow_str_bool_translation and value == "false":
altered_value = False
@@ -625,71 +580,16 @@ def _handle_single_var_pattern(orig_variable: str, var_value_and_file_map: Dict[
var_value = _handle_indexing(orig_variable[6:], lambda r: locals_values.get(r))
if var_value is not None:
return var_value
- elif orig_variable.startswith("to") and orig_variable.endswith(")"):
- # https://www.terraform.io/docs/configuration/functions/tobool.html
- if orig_variable.startswith("tobool("):
- bool_variable = orig_variable[7:-1].lower()
- bool_value = convert_str_to_bool(bool_variable)
- if isinstance(bool_value, bool):
- return bool_value
- else:
- return orig_variable
- # https://www.terraform.io/docs/configuration/functions/tolist.html
- elif orig_variable.startswith("tolist("):
- altered_value = _eval_string(orig_variable[7:-1])
- if altered_value is None:
- return orig_variable
- return altered_value if isinstance(altered_value, list) else list(altered_value)
- # NOTE: tomap as handled outside this loop (see below)
- # https://www.terraform.io/docs/configuration/functions/tonumber.html
- elif orig_variable.startswith("tonumber("):
- num_variable = orig_variable[9:-1]
- if num_variable.startswith('"') and num_variable.endswith('"'):
- num_variable = num_variable[1:-1]
- try:
- if "." in num_variable:
- return float(num_variable)
- else:
- return int(num_variable)
- except ValueError:
- return orig_variable
- # https://www.terraform.io/docs/configuration/functions/toset.html
- elif orig_variable.startswith("toset("):
- altered_value = _eval_string(orig_variable[6:-1])
- if altered_value is None:
- return orig_variable
- return set(altered_value)
- # https://www.terraform.io/docs/configuration/functions/tostring.html
- elif orig_variable.startswith("tostring("):
- altered_value = orig_variable[9:-1]
- # Indicates a safe string, all good
- if altered_value.startswith('"') and altered_value.endswith('"'):
- return altered_value[1:-1]
- # Otherwise, need to check for valid types (number or bool)
- bool_value = convert_str_to_bool(altered_value)
- if isinstance(bool_value, bool):
- return bool_value
- else:
- try:
- if "." in altered_value:
- return str(float(altered_value))
- else:
- return str(int(altered_value))
- except ValueError:
- return orig_variable # no change
- elif orig_variable.startswith("merge(") and orig_variable.endswith(")"):
- altered_value = orig_variable[6:-1]
- args = split_merge_args(altered_value)
- if args is None:
- return orig_variable
- merged_map = {}
- for arg in args:
- if arg.startswith("{"):
- value = _map_string_to_native(arg)
- if value is None:
- return orig_variable
- else:
- value = _handle_single_var_pattern(arg,
+ elif orig_variable.endswith(")") and not orig_variable.startswith("(") and "(" in orig_variable:
+ # Function handling. See parser_functions for implementations.
+ paren_start = orig_variable.index("(")
+ function_name = orig_variable[:paren_start]
+ function = getattr(parser_functions, # call from `parser_functions` script
+ function_name, # name of the function which will be executed
+ function_nyi_handler) # function (in this file) called if lookup fails
+ value_before = orig_variable[paren_start + 1: -1]
+ value_after = function(value_before,
+ var_resolver=lambda v: _handle_single_var_pattern(v,
var_value_and_file_map,
locals_values,
resource_list,
@@ -697,17 +597,10 @@ def _handle_single_var_pattern(orig_variable: str, var_value_and_file_map: Dict[
module_data_retrieval,
eval_map_by_var_name,
context,
- arg,
- root_directory)
- if isinstance(value, dict):
- merged_map.update(value)
- else:
- return orig_variable # don't know what this is, blow out
- return merged_map
- # TODO - format() support, still in progress
- # elif orig_variable.startswith("format(") and orig_variable.endswith(")"):
- # format_tokens = orig_variable[7:-1].split(",")
- # return format_tokens[0].format([_to_native_value(t) for t in format_tokens[1:]])
+ v,
+ root_directory),
+ function_name=function_name)
+ return value_after if value_after != FUNCTION_FAILED else orig_variable
elif _RESOURCE_REF_PATTERN.match(orig_variable):
# Reference to resources, example: 'aws_s3_bucket.example.bucket'
@@ -823,36 +716,11 @@ def _clean_bad_definitions(tf_definition_list):
}
-def _eval_string(value: str) -> Optional[Any]:
- try:
- value_string = value.replace("'", '"')
- parsed = hcl2.loads(f'eval = {value_string}\n') # NOTE: newline is needed
- return parsed["eval"][0]
- except Exception:
- return None
-
-
def _to_native_value(value: str) -> Any:
if value.startswith('"') or value.startswith("'"):
return value[1:-1]
else:
- return _eval_string(value)
-
-
-def _tostring(value: Any) -> str:
- if value is True:
- return "true"
- elif value is False:
- return "false"
- return str(value)
-
-
-def _map_string_to_native(value: str) -> Optional[Dict]:
- try:
- value_string = value.replace("'", '"')
- return json.loads(value_string)
- except Exception:
- return None
+ return eval_string(value)
def _remove_module_dependency_in_path(path):
@@ -904,3 +772,10 @@ def _safe_index(sequence_hopefully, index) -> Optional[Any]:
logging.debug(f'Failed to parse index int ({index}) out of {sequence_hopefully}')
logging.debug(e, stack_info=True)
return None
+
+
+def function_nyi_handler(original, function_name, **_):
+ logging.debug("Function '%s' is not yet implemented and will not be handled. Please file a"
+ "feature request if it is important to your evaluation (value: '%s')",
+ function_name, original)
+ return FUNCTION_FAILED

ADD
checkov/terraform/parser_functions.py
+import logging
+
+from checkov.common.util.type_forcers import convert_str_to_bool
+from checkov.terraform.parser_utils import eval_string, split_merge_args, string_to_native, to_string
+
+#
+# Functions defined in this file implement terraform functions.
+#
+# Inputs:
+# - First arg (unnamed) - the value string provided to the function
+# - "var_resolver" - function pointer to resolve variable/local references and such
+# - "function_name" - name of the function being called (mainly useful for error reporting when a
+# function isn't defined)
+# These may be expanded over time, so accepting kwargs (via `**`) is recommended.
+#
+# If the value cannot be processed, `FUNCTION_FAILED` should be returned.
+#
+
+FUNCTION_FAILED = "____FUNCTION_FAILED____"
+
+
+def merge(original, var_resolver, **_):
+ args = split_merge_args(original)
+ if args is None:
+ return FUNCTION_FAILED
+ merged_map = {}
+ for arg in args:
+ if arg.startswith("{"):
+ arg_value = string_to_native(arg)
+ if arg_value is None:
+ return FUNCTION_FAILED
+ else:
+ arg_value = var_resolver(arg)
+ if isinstance(arg_value, dict):
+ merged_map.update(arg_value)
+ else:
+ return FUNCTION_FAILED # don't know what this is, blow out
+ return merged_map
+
+
+def concat(original, var_resolver, **_):
+ args = split_merge_args(original)
+ if args is None:
+ return FUNCTION_FAILED
+ merged_list = []
+ for arg in args:
+ if arg.startswith("["):
+ value = eval_string(arg)
+ if value is None:
+ logging.debug("Unable to convert to list: %s", arg)
+ return FUNCTION_FAILED
+ else:
+ value = var_resolver(arg)
+ if isinstance(value, list):
+ merged_list.extend(value)
+ else:
+ return FUNCTION_FAILED # don't know what this is, blow out
+ return merged_list
+
+
+def tobool(original, **_):
+ # https://www.terraform.io/docs/configuration/functions/tobool.html
+ bool_value = convert_str_to_bool(original)
+ return bool_value if isinstance(bool_value, bool) else FUNCTION_FAILED
+
+
+def tonumber(original, **_):
+ # https://www.terraform.io/docs/configuration/functions/tonumber.html
+ if original.startswith('"') and original.endswith('"'):
+ original = original[1:-1]
+ try:
+ if "." in original:
+ return float(original)
+ else:
+ return int(original)
+ except ValueError:
+ return FUNCTION_FAILED
+
+
+def tostring(original, **_):
+ # Indicates a safe string, all good
+ if original.startswith('"') and original.endswith('"'):
+ return original[1:-1]
+ # Otherwise, need to check for valid types (number or bool)
+ bool_value = convert_str_to_bool(original)
+ if isinstance(bool_value, bool):
+ return bool_value
+ else:
+ try:
+ if "." in original:
+ return str(float(original))
+ else:
+ return str(int(original))
+ except ValueError:
+ return FUNCTION_FAILED # no change
+
+
+def tolist(original, **_):
+ # https://www.terraform.io/docs/configuration/functions/tolist.html
+ altered_value = eval_string(original)
+ if altered_value is None:
+ return FUNCTION_FAILED
+ return altered_value if isinstance(altered_value, list) else list(altered_value)
+
+
+def toset(original, **_):
+ # https://www.terraform.io/docs/configuration/functions/toset.html
+ altered_value = eval_string(original)
+ if altered_value is None:
+ return FUNCTION_FAILED
+ return altered_value if isinstance(altered_value, set) else set(altered_value)
+
+
+def tomap(original, **_):
+ original = original.replace(":", "=") # converted to colons by parser #shrug
+
+ altered_value = eval_string(original)
+ if altered_value is None or not isinstance(altered_value, dict):
+ return FUNCTION_FAILED
+
+ # If there is a string and anything else, convert to string
+ had_string = False
+ had_something_else = False
+ for k, v in altered_value.items():
+ if v == "${True}":
+ altered_value[k] = True
+ v = True
+ elif v == "${False}":
+ altered_value[k] = False
+ v = False
+
+ if isinstance(v, str):
+ had_string = True
+ if had_something_else:
+ break
+ else:
+ had_something_else = True
+ if had_string:
+ break
+ if had_string and had_something_else:
+ return {k: to_string(v) for k, v in altered_value.items()}
+ return altered_value

RENAME
checkov/terraform/parser_var_blocks.py
checkov/terraform/parser_utils.py
+import json
from dataclasses import dataclass
from enum import Enum, IntEnum
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
+
+import hcl2
@dataclass
@@ -221,3 +224,27 @@ def _str_parser_loop_collection_helper(c: str, inside_collection_stack: List[str
inside_collection_stack.insert(0, ")")
return processing_str_escape
+
+
+def eval_string(value: str) -> Optional[Any]:
+ try:
+ value_string = value.replace("'", '"')
+ parsed = hcl2.loads(f'eval = {value_string}\n') # NOTE: newline is needed
+ return parsed["eval"][0]
+ except Exception:
+ return None
+
+
+def string_to_native(value: str) -> Optional[Dict]:
+ try:
+ value_string = value.replace("'", '"')
+ return json.loads(value_string)
+ except Exception:
+ return None
+
+def to_string(value: Any) -> str:
+ if value is True:
+ return "true"
+ elif value is False:
+ return "false"
+ return str(value)
\ No newline at end of file

ADD
tests/terraform/parser/resources/parser_scenarios/bogus_function/expected.json
+{
+ "main.tf": {
+ "value": ["${not_a_real_function(1,2,3,4)}"]
+ }
+}
\ No newline at end of file

ADD
tests/terraform/parser/resources/parser_scenarios/bogus_function/main.tf
+value = not_a_real_function(1, 2, 3, 4)
\ No newline at end of file

ADD
tests/terraform/parser/resources/parser_scenarios/concat_function/expected.json
+{
+ "main.tf": {
+ "locals": [
+ {
+ "inline_map": [
+ {
+ "key": "a_key",
+ "value": "a_value",
+ "propagate_at_launch": false
+ }
+ ]
+ },
+ {
+ "simple_list": [
+ [
+ "a",
+ "",
+ "b",
+ "c"
+ ]
+ ],
+ "simple_list2": [
+ [
+ "a",
+ "",
+ "b",
+ "c"
+ ]
+ ]
+ }
+ ],
+ "variable": [
+ {
+ "extra_tags": {
+ "default": [
+ [
+ {
+ "key": "Foo",
+ "value": "Bar",
+ "propagate_at_launch": true
+ },
+ {
+ "key": "Baz",
+ "value": "Bam",
+ "propagate_at_launch": true
+ }
+ ]
+ ]
+ }
+ }
+ ],
+ "resource": [
+ {
+ "aws_autoscaling_group": {
+ "bar": {
+ "name": [
+ "foobar3-terraform-test"
+ ],
+ "max_size": [
+ 5
+ ],
+ "min_size": [
+ 2
+ ],
+ "tags": [
+ [
+ {
+ "key": "interpolation1",
+ "value": "value3",
+ "propagate_at_launch": true
+ },
+ {
+ "key": "interpolation2",
+ "value": "value4",
+ "propagate_at_launch": true
+ },
+ {
+ "key": "a_key",
+ "value": "a_value",
+ "propagate_at_launch": false
+ },
+ {
+ "key": "Foo",
+ "value": "Bar",
+ "propagate_at_launch": true
+ },
+ {
+ "key": "Baz",
+ "value": "Bam",
+ "propagate_at_launch": true
+ }
+ ]
+ ]
+ }
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file

ADD
tests/terraform/parser/resources/parser_scenarios/concat_function/main.tf
+# Loosely from https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group
+locals {
+ inline_map = {
+ key = "a_key"
+ value = "a_value"
+ propagate_at_launch = false
+ }
+}
+
+variable "extra_tags" {
+ default = [
+ {
+ key = "Foo"
+ value = "Bar"
+ propagate_at_launch = true
+ },
+ {
+ key = "Baz"
+ value = "Bam"
+ propagate_at_launch = true
+ }
+ ]
+}
+
+resource "aws_autoscaling_group" "bar" {
+ name = "foobar3-terraform-test"
+ max_size = 5
+ min_size = 2
+
+ tags = concat(
+ [
+ {
+ "key" = "interpolation1"
+ "value" = "value3"
+ "propagate_at_launch" = true
+ },
+ {
+ "key" = "interpolation2"
+ "value" = "value4"
+ "propagate_at_launch" = true
+ },
+ local.inline_map
+ ],
+ var.extra_tags,
+ )
+}
+
+# From https://www.terraform.io/docs/language/functions/concat.html
+locals {
+ simple_list = concat(["a", ""], ["b", "c"])
+ simple_list2 = concat(["a"], [""], ["b"], ["c"])
+}
\ No newline at end of file

MODIFY
tests/terraform/parser/test_parser_internals.py
tests/terraform/parser/test_parser_internals.py
@@ -6,4 +6,4 @@ from checkov.terraform import parser
class TestParserInternals(unittest.TestCase):
def test_eval_string_to_list(self):
expected = ["a", "b", "c"]
- assert parser._eval_string('["a", "b", "c"]') == expected
+ assert parser.eval_string('["a", "b", "c"]') == expected

MODIFY
tests/terraform/parser/test_parser_scenarios.py
tests/terraform/parser/test_parser_scenarios.py
@@ -40,6 +40,9 @@ class TestParserScenarios(unittest.TestCase):
def test_compound_local(self):
self.go("compound_local")
+ def test_concat_function(self):
+ self.go("concat_function")
+
def test_merge_function(self):
self.go("merge_function")
@@ -124,6 +127,9 @@ class TestParserScenarios(unittest.TestCase):
def test_account_dirs_and_modules(self):
self.go("account_dirs_and_modules")
+ def test_bogus_function(self):
+ self.go("bogus_function")
+
@staticmethod
def go(dir_name):
dir_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),

MODIFY
tests/terraform/parser/test_parser_var_blocks.py
tests/terraform/parser/test_parser_var_blocks.py
@@ -3,7 +3,7 @@ import unittest
from typing import List, Tuple
-from checkov.terraform.parser_var_blocks import VarBlockMatch as VBM, split_merge_args, find_var_blocks
+from checkov.terraform.parser_utils import VarBlockMatch as VBM, split_merge_args, find_var_blocks
class TestParserInternals(unittest.TestCase):

Support for the TF `concat` function (#922)

In addition, function handling has been reworked to be more concise and allow easier addition of new functions.