diff --git a/scripts/lib/CIME/case/case.py b/scripts/lib/CIME/case/case.py
index e35b576da..d6f66c29f 100644
--- a/scripts/lib/CIME/case/case.py
+++ b/scripts/lib/CIME/case/case.py
@@ -1729,8 +1729,11 @@ directory, NOT in this subdirectory."""
         lines_len = len(lines)
         lines.insert(lines_len-1 if init else lines_len, "{}\n\n".format(cmd))
 
-        with open(os.path.join(caseroot, "replay.sh"), "a") as fd:
-            fd.writelines(lines)
+        try:
+            with open(os.path.join(caseroot, "replay.sh"), "a") as fd:
+                fd.writelines(lines)
+        except PermissionError:
+            logger.warning("Could not write to 'replay.sh' script")
 
     def create(self, casename, srcroot, compset_name, grid_name,
                user_mods_dirs=None, machine_name=None,
diff --git a/scripts/lib/CIME/provenance.py b/scripts/lib/CIME/provenance.py
index 6db56b6a6..30b0fc987 100644
--- a/scripts/lib/CIME/provenance.py
+++ b/scripts/lib/CIME/provenance.py
@@ -481,6 +481,7 @@ def _save_postrun_timing_e3sm(case, lid):
     globs_to_copy.append("timing/*.{}*".format(lid))
     globs_to_copy.append("CaseStatus")
     globs_to_copy.append(os.path.join(rundir, "spio_stats.{}.tar.gz".format(lid)))
+    globs_to_copy.append(os.path.join(caseroot, "replay.sh"))
     # Can't use a single glob, similar files e.g. {filename}.{lid} get picked up.
     bld_filenames = ["GIT_STATUS", "GIT_DIFF", "GIT_LOG", "GIT_REMOTE",
                      "GIT_CONFIG", "GIT_SUBMODULE_STATUS"]

diff --git a/scripts/lib/CIME/tests/test_case.py b/scripts/lib/CIME/tests/test_case.py
index 6595b861b..8585fb2dd 100644
--- a/scripts/lib/CIME/tests/test_case.py
+++ b/scripts/lib/CIME/tests/test_case.py
@@ -216,6 +216,25 @@ class TestCase_RecordCmd(unittest.TestCase):
 
         for x, y in zip(calls, expected):
             self.assertTrue(x == y, calls)
+    @mock.patch("CIME.case.case.Case.__init__", return_value=None)
+    @mock.patch("CIME.case.case.Case.flush")
+    @mock.patch("CIME.case.case.Case.get_value")
+    @mock.patch("CIME.case.case.open", mock.mock_open())
+    @mock.patch("time.strftime", return_value="00:00:00")
+    @mock.patch("sys.argv", ["/src/create_newcase"])
+    def test_error(self, strftime, get_value, flush, init): # pylint: disable=unused-argument
+        Case._force_read_only = False # pylint: disable=protected-access
+
+        with self.tempdir as tempdir, mock.patch("CIME.case.case.open", mock.mock_open()) as m:
+            m.side_effect = PermissionError()
+
+            with Case(tempdir) as case:
+                get_value.side_effect = [
+                    tempdir,
+                    "/src"
+                ]
+
+                case.record_cmd()
 
     @mock.patch("CIME.case.case.Case.__init__", return_value=None)
     @mock.patch("CIME.case.case.Case.flush")
diff --git a/scripts/lib/CIME/tests/test_provenance.py b/scripts/lib/CIME/tests/test_provenance.py
index 89b619779..0263bac2d 100644
--- a/scripts/lib/CIME/tests/test_provenance.py
+++ b/scripts/lib/CIME/tests/test_provenance.py
@@ -1,125 +1,94 @@
 import os
 import sys
 import unittest
+from unittest import mock
 
 from CIME import provenance
 
-from . import utils
-
 class TestProvenance(unittest.TestCase):
-    def test_run_git_cmd_recursively(self):
-        with utils.Mocker() as mock:
-            open_mock = mock.patch(
-                "builtins.open" if sys.version_info.major > 2 else
-                    "__builtin__.open",
-                ret=utils.Mocker()
-            )
-            provenance.run_cmd = utils.Mocker(return_value=(0, "data", None))
-            provenance._run_git_cmd_recursively('status', '/srcroot', '/output.txt') # pylint: disable=protected-access
-
-        self.assertTrue(
-            open_mock.calls[0]["args"] == ("/output.txt", "w"),
-            open_mock.calls
-        )
-
-        write = open_mock.ret.method_calls["write"]
-
-        self.assertTrue(write[0]["args"][0] == "data\n\n", write)
-        self.assertTrue(write[1]["args"][0] == "data\n", write)
-
-        run_cmd = provenance.run_cmd.calls
-
-        self.assertTrue(run_cmd[0]["args"][0] == "git status")
-        self.assertTrue(run_cmd[0]["kwargs"]["from_dir"] == "/srcroot")
-
-        self.assertTrue(run_cmd[1]["args"][0] == "git submodule foreach"
-                        " --recursive \"git status; echo\"", run_cmd)
-        self.assertTrue(run_cmd[1]["kwargs"]["from_dir"] == "/srcroot")
-
-    def test_run_git_cmd_recursively_error(self):
-        with utils.Mocker() as mock:
-            open_mock = mock.patch(
-                "builtins.open" if sys.version_info.major > 2 else
-                    "__builtin__.open",
-                ret=utils.Mocker()
-            )
-            provenance.run_cmd = utils.Mocker(return_value=(1, "data", "error"))
-            provenance._run_git_cmd_recursively('status', '/srcroot', '/output.txt') # pylint: disable=protected-access
-
-        write = open_mock.ret.method_calls["write"]
-
-        self.assertTrue(write[0]["args"][0] == "error\n\n", write)
-        self.assertTrue(write[1]["args"][0] == "error\n", write)
-
-        run_cmd = provenance.run_cmd.calls
-
-        self.assertTrue(run_cmd[0]["args"][0] == "git status")
-        self.assertTrue(run_cmd[0]["kwargs"]["from_dir"] == "/srcroot")
-
-        self.assertTrue(run_cmd[1]["args"][0] == "git submodule foreach"
-                        " --recursive \"git status; echo\"", run_cmd)
-        self.assertTrue(run_cmd[1]["kwargs"]["from_dir"] == "/srcroot")
-
-    def test_record_git_provenance(self):
-        with utils.Mocker() as mock:
-            open_mock = mock.patch(
-                "builtins.open" if sys.version_info.major > 2 else
-                    "__builtin__.open",
-                ret=utils.Mocker()
-            )
-
-            provenance.safe_copy = utils.Mocker()
-            provenance.run_cmd = utils.Mocker(return_value=(0, "data", None))
-            provenance._record_git_provenance("/srcroot", "/output", "5") # pylint: disable=protected-access
+    @mock.patch("CIME.provenance.run_cmd")
+    def test_run_git_cmd_recursively(self, run_cmd):
+        run_cmd.return_value = (0, "data", None)
+
+        with mock.patch("CIME.provenance.open", mock.mock_open()) as m:
+            provenance._run_git_cmd_recursively("status", "/srcroot", "/output.txt") # pylint: disable=protected-access
+
+        m.assert_called_with("/output.txt", "w")
+
+        write = m.return_value.__enter__.return_value.write
+
+        write.assert_any_call("data\n\n")
+        write.assert_any_call("data\n")
+
+        run_cmd.assert_any_call("git status", from_dir="/srcroot")
+        run_cmd.assert_any_call(
+            "git submodule foreach --recursive \"git status; echo\"",
+            from_dir="/srcroot")
 
-        expected = [
-            ("/output/GIT_STATUS.5", "w"),
-            ("/output/GIT_DIFF.5", "w"),
-            ("/output/GIT_LOG.5", "w"),
-            ("/output/GIT_REMOTE.5", "w")
-        ]
+    @mock.patch("CIME.provenance.run_cmd")
+    def test_run_git_cmd_recursively_error(self, run_cmd):
+        run_cmd.return_value = (1, "data", "error")
 
-        for i in range(4):
-            self.assertTrue(
-                open_mock.calls[i]["args"] == expected[i],
-                open_mock.calls
-            )
+        with mock.patch("CIME.provenance.open", mock.mock_open()) as m:
+            provenance._run_git_cmd_recursively("status", "/srcroot", "/output.txt") # pylint: disable=protected-access
 
-        write = open_mock.ret.method_calls["write"]
+        m.assert_called_with("/output.txt", "w")
 
-        expected = [
-            "data\n\n",
-            "data\n",
-        ]
+        write = m.return_value.__enter__.return_value.write
 
-        for x in range(8):
-            self.assertTrue(write[x]["args"][0] == expected[x%2], write)
+        write.assert_any_call("error\n\n")
+        write.assert_any_call("error\n")
 
-        run_cmd = provenance.run_cmd.calls
+        run_cmd.assert_any_call("git status", from_dir="/srcroot")
+        run_cmd.assert_any_call(
+            "git submodule foreach --recursive \"git status; echo\"",
+            from_dir="/srcroot")
+
+    @mock.patch("CIME.provenance.safe_copy")
+    @mock.patch("CIME.provenance.run_cmd")
+    def test_record_git_provenance(self, run_cmd, safe_copy):
+        run_cmd.return_value = (0, "data", None)
+
+        with mock.patch("CIME.provenance.open", mock.mock_open()) as m:
+            provenance._record_git_provenance("/srcroot", "/output", "5") # pylint: disable=protected-access
 
-        expected = [
-            "git status",
+        m.assert_any_call("/output/GIT_STATUS.5", "w")
+        m.assert_any_call("/output/GIT_DIFF.5", "w")
+        m.assert_any_call("/output/GIT_LOG.5", "w")
+        m.assert_any_call("/output/GIT_REMOTE.5", "w")
+
+        write = m.return_value.__enter__.return_value.write
+
+        write.assert_any_call("data\n\n")
+        write.assert_any_call("data\n")
+
+        run_cmd.assert_any_call("git status", from_dir="/srcroot")
+        run_cmd.assert_any_call(
             "git submodule foreach --recursive \"git status; echo\"",
+            from_dir="/srcroot")
+        run_cmd.assert_any_call(
             "git diff",
+            from_dir="/srcroot")
+        run_cmd.assert_any_call(
             "git submodule foreach --recursive \"git diff; echo\"",
+            from_dir="/srcroot")
+        run_cmd.assert_any_call(
             "git log --first-parent --pretty=oneline -n 5",
+            from_dir="/srcroot")
+        run_cmd.assert_any_call(
             "git submodule foreach --recursive \"git log --first-parent"
-                " --pretty=oneline -n 5; echo\"",
+            " --pretty=oneline -n 5; echo\"",
+            from_dir="/srcroot")
+        run_cmd.assert_any_call(
             "git remote -v",
+            from_dir="/srcroot")
+        run_cmd.assert_any_call(
             "git submodule foreach --recursive \"git remote -v; echo\"",
-        ]
-
-        for x in range(len(run_cmd)):
-            self.assertTrue(run_cmd[x]["args"][0] == expected[x], run_cmd[x])
-
-        self.assertTrue(
-            provenance.safe_copy.calls[0]["args"][0] == "/srcroot/.git/config",
-            provenance.safe_copy.calls
-        )
-        self.assertTrue(
-            provenance.safe_copy.calls[0]["args"][1] == "/output/GIT_CONFIG.5",
-            provenance.safe_copy.calls
-        )
+            from_dir="/srcroot")
+
+        safe_copy.assert_any_call("/srcroot/.git/config",
+                                  "/output/GIT_CONFIG.5",
+                                  preserve_meta=False)
 
 if __name__ == '__main__':
     sys.path.insert(0, os.path.abspath(os.path.join('.', '..', '..', 'lib')))

replay.sh: Capture script in provenance.py for archiving (E3SM)
We need to archive replay.sh in performance archive so that it can be picked up by PACE for E3SM.
Logic needs to be added to https://github.com/ESMCI/cime/blob/master/scripts/lib/CIME/provenance.py#L400

cc @jasonb5 @rljacob 
scripts/lib/CIME/tests/test_case.py::TestCase_RecordCmd::test_error
scripts/lib/CIME/tests/test_case.py::TestCaseSubmit::test__submit scripts/lib/CIME/tests/test_case.py::TestCaseSubmit::test_check_case scripts/lib/CIME/tests/test_case.py::TestCaseSubmit::test_submit scripts/lib/CIME/tests/test_case.py::TestCase::test_copy scripts/lib/CIME/tests/test_case.py::TestCase::test_create scripts/lib/CIME/tests/test_case.py::TestCase::test_new_hash scripts/lib/CIME/tests/test_case.py::TestCase_RecordCmd::test_cmd_arg scripts/lib/CIME/tests/test_case.py::TestCase_RecordCmd::test_init scripts/lib/CIME/tests/test_case.py::TestCase_RecordCmd::test_sub_relative scripts/lib/CIME/tests/test_provenance.py::TestProvenance::test_record_git_provenance scripts/lib/CIME/tests/test_provenance.py::TestProvenance::test_run_git_cmd_recursively scripts/lib/CIME/tests/test_provenance.py::TestProvenance::test_run_git_cmd_recursively_error