diff --git a/fairlearn/metrics/_extra_metrics.py b/fairlearn/metrics/_extra_metrics.py
index 474116e..e970f28 100644
--- a/fairlearn/metrics/_extra_metrics.py
+++ b/fairlearn/metrics/_extra_metrics.py
@@ -61,7 +61,7 @@ def _get_labels_for_confusion_matrix(labels, pos_label):
     # Ensure unique_labels has two elements
     if len(unique_labels) == 1:
         if unique_labels[0] == pos_label:
-            unique_labels = [None, pos_label]
+            unique_labels = [np.iinfo(np.int64).min, pos_label]
         else:
             unique_labels.append(pos_label)
     elif len(unique_labels) == 2:

diff --git a/test/unit/metrics/test_extra_metrics.py b/test/unit/metrics/test_extra_metrics.py
index f26cbba..4f68dc1 100644
--- a/test/unit/metrics/test_extra_metrics.py
+++ b/test/unit/metrics/test_extra_metrics.py
@@ -36,17 +36,17 @@ class TestGetLabelsForConfusionMatrix:
         r1 = _get_labels_for_confusion_matrix([-1], None)
         assert np.array_equal(r1, [-1, 1])
         r2 = _get_labels_for_confusion_matrix([1], None)
-        assert np.array_equal(r2, [None, 1])
+        assert np.array_equal(r2, [np.iinfo(np.int64).min, 1])
 
     def test_single_value_numeric_pos_label(self):
         r0 = _get_labels_for_confusion_matrix([0], 3)
         assert np.array_equal(r0, [0, 3])
         r1 = _get_labels_for_confusion_matrix([0], 0)
-        assert np.array_equal(r1, [None, 0])
+        assert np.array_equal(r1, [np.iinfo(np.int64).min, 0])
 
     def test_single_value_alpha_pos_label(self):
         r0 = _get_labels_for_confusion_matrix(['a'], 'a')
-        assert np.array_equal(r0, [None, 'a'])
+        assert np.array_equal(r0, [np.iinfo(np.int64).min, 'a'])
         r1 = _get_labels_for_confusion_matrix(['a'], 0)
         assert np.array_equal(r1, ['a', 0])
 

BUG: sklearn.confusion_matrix does not work with None anymore
#### Describe the bug
The following tests fail in the marked lines: 
- https://github.com/fairlearn/fairlearn/blob/94229647dda802a59bcbfe7f6d64295d64e96efa/test/unit/metrics/test_extra_metrics.py#L420
- https://github.com/fairlearn/fairlearn/blob/94229647dda802a59bcbfe7f6d64295d64e96efa/test/unit/metrics/test_extra_metrics.py#L388
- https://github.com/fairlearn/fairlearn/blob/94229647dda802a59bcbfe7f6d64295d64e96efa/test/unit/metrics/test_extra_metrics.py#L379

This build shows the errors in full detail https://dev.azure.com/fairlearn/Fairlearn/_build/results?buildId=508&view=logs&j=011e1ec8-6569-5e69-4f06-baf193d1351e&t=e41ce7ed-8506-5fe7-0eaa-68d3583c0fff&l=670

As far as I can tell from looking at the error output it seems like sklearn.confusion_matrix is struggling to compare the returned `None` from `unique_labels = [None, pos_label]` with the present values.
```
TypeError: '<' not supported between instances of 'int' and 'NoneType'
``` 
This behavior probably changed in the latest scikit-learn release. We should fix this asap since it's possible people hit this issue after upgrading to latest versions (or with fresh installation), and our tests are failing which prevents us from merging PRs.

#### Steps/Code to Reproduce
Just run `python -m pytest test/unit/metrics` to reproduce

#### Versions
```
System:
    python: 3.8.10 (default, Jun  2 2021, 10:49:15)  [GCC 9.4.0]
executable: /usr/bin/python
   machine: Linux-5.11.0-34-generic-x86_64-with-glibc2.29

Python dependencies:
    Cython: None
matplotlib: 3.4.3
     numpy: 1.21.2
    pandas: 1.3.3
       pip: 20.0.2
     scipy: 1.7.1
setuptools: 45.2.0
   sklearn: 1.0
    tempeh: None
```


FYI @riedgar-ms who wrote a bunch of these tests and @adrinjalali who may know what's wrong with `confusion_matrix` (or rather our way of using it).
test/unit/metrics/test_extra_metrics.py::TestGetLabelsForConfusionMatrix::test_single_value_numeric_no_pos_label test/unit/metrics/test_extra_metrics.py::TestGetLabelsForConfusionMatrix::test_single_value_numeric_pos_label test/unit/metrics/test_extra_metrics.py::TestGetLabelsForConfusionMatrix::test_single_value_alpha_pos_label test/unit/metrics/test_extra_metrics.py::TestSingleValueArrays::test_all_zeros test/unit/metrics/test_extra_metrics.py::TestSingleValueArrays::test_all_ones test/unit/metrics/test_extra_metrics.py::TestSingleValueArrays::test_all_negative_ones
test/unit/metrics/test_extra_metrics.py::TestGetLabelsForConfusionMatrix::test_smoke test/unit/metrics/test_extra_metrics.py::TestGetLabelsForConfusionMatrix::test_smoke_numeric_pos_label test/unit/metrics/test_extra_metrics.py::TestGetLabelsForConfusionMatrix::test_smoke_alpha_pos_label test/unit/metrics/test_extra_metrics.py::TestGetLabelsForConfusionMatrix::test_too_many_values test/unit/metrics/test_extra_metrics.py::TestGetLabelsForConfusionMatrix::test_need_pos_label test/unit/metrics/test_extra_metrics.py::TestGetLabelsForConfusionMatrix::test_pos_label_not_in_data test/unit/metrics/test_extra_metrics.py::TestTPR::test_all_correct test/unit/metrics/test_extra_metrics.py::TestTPR::test_none_correct test/unit/metrics/test_extra_metrics.py::TestTPR::test_some_correct test/unit/metrics/test_extra_metrics.py::TestTPR::test_against_sklearn test/unit/metrics/test_extra_metrics.py::TestTPR::test_against_sklearn_weighted test/unit/metrics/test_extra_metrics.py::TestTPR::test_tpr_values_alpha test/unit/metrics/test_extra_metrics.py::TestTNR::test_all_correct test/unit/metrics/test_extra_metrics.py::TestTNR::test_none_correct test/unit/metrics/test_extra_metrics.py::TestTNR::test_some_correct test/unit/metrics/test_extra_metrics.py::TestTNR::test_some_correct_other_labels test/unit/metrics/test_extra_metrics.py::TestTNR::test_tnr_some_correct_with_false_negative test/unit/metrics/test_extra_metrics.py::TestTNR::test_against_sklearn test/unit/metrics/test_extra_metrics.py::TestTNR::test_against_sklearn_weighted test/unit/metrics/test_extra_metrics.py::TestFNR::test_all_correct test/unit/metrics/test_extra_metrics.py::TestFNR::test_none_correct test/unit/metrics/test_extra_metrics.py::TestFNR::test_with_false_positive test/unit/metrics/test_extra_metrics.py::TestFNR::test_some_correct test/unit/metrics/test_extra_metrics.py::TestFNR::test_some_correct_other_labels test/unit/metrics/test_extra_metrics.py::TestFNR::test_against_sklearn test/unit/metrics/test_extra_metrics.py::TestFNR::test_against_sklearn_weighted test/unit/metrics/test_extra_metrics.py::TestFPR::test_all_correct test/unit/metrics/test_extra_metrics.py::TestFPR::test_none_correct test/unit/metrics/test_extra_metrics.py::TestFPR::test_some_correct test/unit/metrics/test_extra_metrics.py::TestFPR::test_against_sklearn test/unit/metrics/test_extra_metrics.py::TestFPR::test_against_sklearn_weighted test/unit/metrics/test_extra_metrics.py::TestCount::test_empty_group test/unit/metrics/test_extra_metrics.py::TestCount::test_group_of_one test/unit/metrics/test_extra_metrics.py::TestCount::test_multi_group test/unit/metrics/test_extra_metrics.py::TestCount::test_unequal_y_sizes