diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2eee6ed..0ae0743 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file, and this pr
 * Fixed annotation message filtering bug per [#145](https://github.com/microsoft/pybryt/issues/145)
 * Added `dataclasses` backport to required modules for Python &lt; 3.7
 * Added a `group` argument to the `check` context manager per [#146](https://github.com/microsoft/pybryt/issues/146)
+* Moved named annotation filtering into `ReferenceImplementation` constructor per [#147](https://github.com/microsoft/pybryt/issues/147)
 
 ## 0.3.1 - 2021-12-01
 
diff --git a/pybryt/annotations/annotation.py b/pybryt/annotations/annotation.py
index f386995..5abfe34 100644
--- a/pybryt/annotations/annotation.py
+++ b/pybryt/annotations/annotation.py
@@ -4,13 +4,12 @@ __all__ = ["Annotation", "AnnotationResult"]
 
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional
 
 from ..execution import is_complexity_tracing_enabled, MemoryFootprint
 
 
 _TRACKED_ANNOTATIONS = []
-_GROUP_INDICES = {}
 _ANNOTATION_COUNTER = 0
 
 
@@ -71,23 +70,12 @@ class Annotation(ABC):
     
     def _track(self) -> None:
         """
-        Tracks this annotation in ``_TRACKED_ANNOTATIONS`` and updates ``_GROUP_INDICES`` with the
-        index of the annotation if ``self.group`` is present. If the annotation has children
+        Tracks this annotation in ``_TRACKED_ANNOTATIONS``. If the annotation has children
         (returned by ``self.children``), the children are removed from ``_TRACKED_ANNOTATIONS``.
         """
         if is_complexity_tracing_enabled():
             return
 
-        global _GROUP_INDICES, _TRACKED_ANNOTATIONS
-
-        idx = len(_TRACKED_ANNOTATIONS)
-        if self.name not in _GROUP_INDICES:
-            _GROUP_INDICES[self.name] = []
-        if self.limit is not None and len(_GROUP_INDICES[self.name]) >= self.limit:
-            return
-        else:
-            _GROUP_INDICES[self.name].append(idx)
-    
         for child in self.children:
             try:
                 _TRACKED_ANNOTATIONS.remove(child)
@@ -112,9 +100,8 @@ class Annotation(ABC):
         Resets the list of tracked annotations and the mapping of group names to indices in that
         list.
         """
-        global _ANNOTATION_COUNTER, _GROUP_INDICES, _TRACKED_ANNOTATIONS
+        global _ANNOTATION_COUNTER, _TRACKED_ANNOTATIONS
         _TRACKED_ANNOTATIONS.clear()
-        _GROUP_INDICES.clear()
         _ANNOTATION_COUNTER = 0
 
     @property
diff --git a/pybryt/reference.py b/pybryt/reference.py
index a635c10..4e0772b 100644
--- a/pybryt/reference.py
+++ b/pybryt/reference.py
@@ -38,7 +38,19 @@ class ReferenceImplementation(Serializable):
         if not all(isinstance(ann, Annotation) for ann in annotations):
             raise TypeError("Found non-annotation in annotations")
         
-        self.annotations = annotations
+        self.annotations, name_counts = [], {}
+        for ann in annotations:
+            track = True
+            if ann.name is not None:
+                count = name_counts.get(ann.name, 0)
+                if ann.limit is None or ann.limit > count:
+                    name_counts[ann.name] = count + 1
+                else:
+                    track = False
+
+            if track:
+                self.annotations.append(ann)
+
         self.name = name
 
     def __eq__(self, other: Any) -> bool:

diff --git a/tests/annotations/test_annotation.py b/tests/annotations/test_annotation.py
index 96f0b14..6653d56 100644
--- a/tests/annotations/test_annotation.py
+++ b/tests/annotations/test_annotation.py
@@ -18,9 +18,8 @@ def test_name_group_limit():
         vs.append(pybryt.Value(val, name="foo", limit=11))
     
     tracked = pybryt.Annotation.get_tracked_annotations()
-    assert len(tracked) == 11, "Too many tracked annotations"
-    assert tracked == vs[:11], "Wrong tracked annotations"
-    assert all(v.name == "foo" and v.limit == 11 for v in vs)
+    assert len(tracked) == 100
+    assert tracked == vs, "Wrong tracked annotations"
 
     res = vs[-1].check(footprint)
     assert_object_attrs(res, {
diff --git a/tests/test_reference.py b/tests/test_reference.py
index e77644b..d8745cc 100644
--- a/tests/test_reference.py
+++ b/tests/test_reference.py
@@ -125,6 +125,21 @@ def test_reference_construction():
         ref2 = more_refs[0]
         assert ref2 == expected_ref2
 
+    # check filtering named annotations (#147)
+    annots = [
+        Value(0),
+        Value(1, name="1"),
+        Value(2, name="1"),
+        Value(3, name="1"),
+        Value(4, name="2", limit=2),
+        Value(5, name="2", limit=2),
+        Value(6, name="2", limit=2),
+        Value(7, name="3", limit=2),
+    ]
+    ref = ReferenceImplementation("named-annotations", annots)
+    assert len(ref.annotations) == 7
+    assert annots[-2] not in ref.annotations
+
 
 def test_construction_errors():
     """

Limiting the number of annotations in a reference only works when compiling a notebook, not when constructing the object manually
Consider the example:

```python
max_ref = []
def maximum(l, track=False):
    m = max(l)
    if track:
        max_ref.append(pybryt.Value(
            m,
            name="list-maximum",
            limit=5,
            success_message="Found the max!", 
            failure_message="Did not find the max",
        ))
    return m

for _ in range(1000):
    test_list = np.random.normal(size=100)
    maximum(test_list, track=True)

max_ref = pybryt.ReferenceImplementation("maximum", max_ref)
max_ref.annotations
```

`max_ref.annotations` here has length 1000 even though the limit is set of 5 for all annotations in the reference. The filtering behavior should be moved into the reference implementation constructor instead of being in `Annotation._track`.
tests/annotations/test_annotation.py::test_name_group_limit tests/test_reference.py::test_reference_construction
tests/annotations/test_annotation.py::test_get_reset_tracked_annotations tests/annotations/test_annotation.py::test_messages tests/annotations/test_annotation.py::test_bitwise_ops tests/test_reference.py::test_construction_errors tests/test_reference.py::test_run_and_results tests/test_reference.py::test_generate_report