diff --git a/altair/utils/core.py b/altair/utils/core.py
index 53785b17..c47b9a04 100644
--- a/altair/utils/core.py
+++ b/altair/utils/core.py
@@ -193,8 +193,6 @@ def infer_vegalite_type(data):
     # Otherwise, infer based on the dtype of the input
     typ = infer_dtype(data)
 
-    # TODO: Once this returns 'O', please update test_select_x and test_select_y in test_api.py
-
     if typ in [
         "floating",
         "mixed-integer-float",
@@ -203,6 +201,8 @@ def infer_vegalite_type(data):
         "complex",
     ]:
         return "quantitative"
+    elif typ == "categorical" and data.cat.ordered:
+        return ("ordinal", data.cat.categories.tolist())
     elif typ in ["string", "bytes", "categorical", "boolean", "mixed", "unicode"]:
         return "nominal"
     elif typ in [
@@ -316,8 +316,9 @@ def sanitize_dataframe(df):  # noqa: C901
 
     for col_name, dtype in df.dtypes.items():
         if str(dtype) == "category":
-            # XXXX: work around bug in to_json for categorical types
+            # Work around bug in to_json for categorical types in older versions of pandas
             # https://github.com/pydata/pandas/issues/10778
+            # https://github.com/altair-viz/altair/pull/2170
             col = df[col_name].astype(object)
             df[col_name] = col.where(col.notnull(), None)
         elif str(dtype) == "string":
@@ -527,6 +528,10 @@ def parse_shorthand(
     if isinstance(data, pd.DataFrame) and "type" not in attrs:
         if "field" in attrs and attrs["field"] in data.columns:
             attrs["type"] = infer_vegalite_type(data[attrs["field"]])
+            # ordered categorical dataframe columns return the type and sort order as a tuple
+            if isinstance(attrs["type"], tuple):
+                attrs["sort"] = attrs["type"][1]
+                attrs["type"] = attrs["type"][0]
     return attrs
 
 
diff --git a/altair/utils/schemapi.py b/altair/utils/schemapi.py
index b9f8bae9..fab24a4a 100644
--- a/altair/utils/schemapi.py
+++ b/altair/utils/schemapi.py
@@ -364,6 +364,14 @@ class SchemaBase(object):
             # parsed_shorthand is removed from context if it exists so that it is
             # not passed to child to_dict function calls
             parsed_shorthand = context.pop("parsed_shorthand", {})
+            # Prevent that pandas categorical data is automatically sorted
+            # when a non-ordinal data type is specifed manually
+            if "sort" in parsed_shorthand and kwds["type"] not in [
+                "ordinal",
+                Undefined,
+            ]:
+                parsed_shorthand.pop("sort")
+
             kwds.update(
                 {
                     k: v
diff --git a/doc/user_guide/encodings/channels.rst b/doc/user_guide/encodings/channels.rst
index 057db9c7..073580b2 100644
--- a/doc/user_guide/encodings/channels.rst
+++ b/doc/user_guide/encodings/channels.rst
@@ -110,7 +110,7 @@ We map the ``symbol`` variable to ``detail`` to use them to group lines.
 Order
 ~~~~~
 
-The `order` option and :class:`Order` channel can sort how marks are drawn on the chart.
+The ``order`` option and :class:`Order` channel can sort how marks are drawn on the chart.
 
 For stacked marks, this controls the order of components of the stack. Here, the elements of each bar are sorted alphabetically by the name of the nominal data in the color channel.
 
@@ -144,9 +144,6 @@ The order can be reversed by changing the sort option to `descending`.
         order=alt.Order("site", sort="descending")
     )
 
-If we want to sort stacked segments in a custom order, we can `follow the approach in this issue comment <https://github.com/altair-viz/altair/issues/245#issuecomment-748443434>`_, although there might be edge cases where this is not fully supported. This also makes the order of the segments align with the order colors shows up in a legend that uses custom sorting for the color domain.
-
-
 The same approach works for other mark types, like stacked areas charts.
 
 .. altair-plot::
@@ -163,7 +160,12 @@ The same approach works for other mark types, like stacked areas charts.
         order=alt.Order("site", sort="ascending")
     )
 
-For line marks, the `order` channel encodes the order in which data points are connected. This can be useful for creating a scatter plot that draws lines between the dots using a different field than the x and y axes.
+Note that unlike the ``sort`` parameter to positional encoding channels,
+the :class:`Order` channel cannot take a list of values to sort by
+and is not automatically sorted when an ordered pandas categorical column is passed.
+If we want to sort stacked segments in a custom order, we can `follow the approach in this issue comment <https://github.com/altair-viz/altair/issues/245#issuecomment-748443434>`_, although there might be edge cases where this is not fully supported. This workaround also makes the order of the segments align with the order that the colors shows up in a legend that uses custom sorting for the color domain.
+
+For line marks, the :class:`Order` channel encodes the order in which data points are connected. This can be useful for creating a scatter plot that draws lines between the dots using a different field than the x and y axes.
 
 .. altair-plot::
 
diff --git a/doc/user_guide/encodings/index.rst b/doc/user_guide/encodings/index.rst
index b46acf6b..32073259 100644
--- a/doc/user_guide/encodings/index.rst
+++ b/doc/user_guide/encodings/index.rst
@@ -318,18 +318,22 @@ Sort Option
 ~~~~~~~~~~~
 
 Some channels accept a  :class:`sort` option which determines the
-order of the scale being used for the channel. There are a number of different
-sort options available:
+order of the scale being used for the channel.
+By default the scale is sorted in ascending alphabetical order,
+unless an `ordered pandas categorical column <https://pandas.pydata.org/docs/user_guide/categorical.html?highlight=categorical#sorting-and-order>`_ is passed (without an explicit type specification)
+in which case Altair will use the column's inherent order to sort the scale.
+There are a number of different
+options available to change the sort order:
 
 - ``sort='ascending'`` (Default) will sort the field's value in ascending order.
-  for string data, this uses standard alphabetical order.
+  For string data, this uses standard alphabetical order.
 - ``sort='descending'`` will sort the field's value in descending order
-- passing the name of an encoding channel to ``sort``, such as ``"x"`` or ``"y"``, allows for
+- Passing the name of an encoding channel to ``sort``, such as ``"x"`` or ``"y"``, allows for
   sorting by that channel. An optional minus prefix can be used for a descending
   sort. For example ``sort='-x'`` would sort by the x channel in descending order.
-- passing a list to ``sort`` allows you to explicitly set the order in which
+- Passing a list to ``sort`` allows you to explicitly set the order in which
   you would like the encoding to appear
-- passing a :class:`EncodingSortField` class to ``sort`` allows you to sort
+- Passing a :class:`EncodingSortField` class to ``sort`` allows you to sort
   an axis by the value of some other field in the dataset.
 
 Here is an example of applying these five different sort approaches on the
diff --git a/tools/schemapi/schemapi.py b/tools/schemapi/schemapi.py
index aeaf156c..d04bdf2d 100644
--- a/tools/schemapi/schemapi.py
+++ b/tools/schemapi/schemapi.py
@@ -362,6 +362,14 @@ class SchemaBase(object):
             # parsed_shorthand is removed from context if it exists so that it is
             # not passed to child to_dict function calls
             parsed_shorthand = context.pop("parsed_shorthand", {})
+            # Prevent that pandas categorical data is automatically sorted
+            # when a non-ordinal data type is specifed manually
+            if "sort" in parsed_shorthand and kwds["type"] not in [
+                "ordinal",
+                Undefined,
+            ]:
+                parsed_shorthand.pop("sort")
+
             kwds.update(
                 {
                     k: v

diff --git a/tests/vegalite/v5/tests/test_api.py b/tests/vegalite/v5/tests/test_api.py
index b2e5aa15..12e8b666 100644
--- a/tests/vegalite/v5/tests/test_api.py
+++ b/tests/vegalite/v5/tests/test_api.py
@@ -123,6 +123,7 @@ def test_chart_infer_types():
             "x": pd.date_range("2012", periods=10, freq="Y"),
             "y": range(10),
             "c": list("abcabcabca"),
+            "s": pd.Categorical([1, 2] * 5, categories=[2, 1], ordered=True),
         }
     )
 
@@ -134,32 +135,45 @@ def test_chart_infer_types():
         assert dct["encoding"]["y"]["field"] == "y"
         assert dct["encoding"]["color"]["type"] == "nominal"
         assert dct["encoding"]["color"]["field"] == "c"
+        assert dct["encoding"]["size"]["type"] == "ordinal"
+        assert dct["encoding"]["size"]["field"] == "s"
+        assert dct["encoding"]["size"]["sort"] == [2, 1]
 
     # Pass field names by keyword
-    chart = alt.Chart(data).mark_point().encode(x="x", y="y", color="c")
+    chart = alt.Chart(data).mark_point().encode(x="x", y="y", color="c", size="s")
     _check_encodings(chart)
 
     # pass Channel objects by keyword
     chart = (
         alt.Chart(data)
         .mark_point()
-        .encode(x=alt.X("x"), y=alt.Y("y"), color=alt.Color("c"))
+        .encode(x=alt.X("x"), y=alt.Y("y"), color=alt.Color("c"), size=alt.Size("s"))
     )
     _check_encodings(chart)
 
     # pass Channel objects by value
-    chart = alt.Chart(data).mark_point().encode(alt.X("x"), alt.Y("y"), alt.Color("c"))
+    chart = (
+        alt.Chart(data)
+        .mark_point()
+        .encode(alt.X("x"), alt.Y("y"), alt.Color("c"), alt.Size("s"))
+    )
     _check_encodings(chart)
 
     # override default types
     chart = (
         alt.Chart(data)
         .mark_point()
-        .encode(alt.X("x", type="nominal"), alt.Y("y", type="ordinal"))
+        .encode(
+            alt.X("x", type="nominal"),
+            alt.Y("y", type="ordinal"),
+            alt.Size("s", type="nominal"),
+        )
     )
     dct = chart.to_dict()
     assert dct["encoding"]["x"]["type"] == "nominal"
     assert dct["encoding"]["y"]["type"] == "ordinal"
+    assert dct["encoding"]["size"]["type"] == "nominal"
+    assert "sort" not in dct["encoding"]["size"]
 
 
 @pytest.mark.parametrize(

Support of ordinal based on pandas' ordered Categorical type?
I've just started to play with altair, using the [diamonds](http://vincentarelbundock.github.io/Rdatasets/datasets.html) dataset. Here is the notebook to clarify what I did https://gist.github.com/pierre-haessig/09fa9268aa0a0e7d91356f681f96ca18

Since, I'm not familiar with altair, I maybe missed something, but I've got the feeling that ordered Categorical types from pandas are not supported.

Indeed, if I use a color='cut' encoding, when cut is a pandas Series with an _ordered_ category dtype, I get by default a nominal type of coloring (with "unordered" colors).

On the other hand, if I force the use of ordered with color='cut:O', I indeed get the ordered colored (the shades of green), but the _order is wrong_! (I get Fair, Good, Ideal, Premium, Very Good, while the correct order is 'Fair', 'Good', 'Very Good', 'Premium', 'Ideal', as manually defined in pandas' category)

tests/vegalite/v5/tests/test_api.py::test_chart_infer_types
tests/vegalite/v5/tests/test_api.py::test_chart_data_types tests/vegalite/v5/tests/test_api.py::test_multiple_encodings[args0-kwargs0] tests/vegalite/v5/tests/test_api.py::test_multiple_encodings[args1-kwargs1] tests/vegalite/v5/tests/test_api.py::test_multiple_encodings[args2-kwargs2] tests/vegalite/v5/tests/test_api.py::test_multiple_encodings[args3-kwargs3] tests/vegalite/v5/tests/test_api.py::test_multiple_encodings[args4-kwargs4] tests/vegalite/v5/tests/test_api.py::test_multiple_encodings[args5-kwargs5] tests/vegalite/v5/tests/test_api.py::test_chart_operations tests/vegalite/v5/tests/test_api.py::test_selection_to_dict tests/vegalite/v5/tests/test_api.py::test_selection_expression tests/vegalite/v5/tests/test_api.py::test_save[altair_saver-html] tests/vegalite/v5/tests/test_api.py::test_save[altair_saver-json] tests/vegalite/v5/tests/test_api.py::test_save[altair_saver-png] tests/vegalite/v5/tests/test_api.py::test_save[altair_saver-svg] tests/vegalite/v5/tests/test_api.py::test_save[altair_saver-pdf] tests/vegalite/v5/tests/test_api.py::test_save[altair_saver-bogus] tests/vegalite/v5/tests/test_api.py::test_save[vl-convert-html] tests/vegalite/v5/tests/test_api.py::test_save[vl-convert-json] tests/vegalite/v5/tests/test_api.py::test_save[vl-convert-pdf] tests/vegalite/v5/tests/test_api.py::test_save[vl-convert-bogus] tests/vegalite/v5/tests/test_api.py::test_save_html[False] tests/vegalite/v5/tests/test_api.py::test_facet_basic tests/vegalite/v5/tests/test_api.py::test_facet_parse tests/vegalite/v5/tests/test_api.py::test_facet_parse_data tests/vegalite/v5/tests/test_api.py::test_selection tests/vegalite/v5/tests/test_api.py::test_transforms tests/vegalite/v5/tests/test_api.py::test_filter_transform_selection_predicates tests/vegalite/v5/tests/test_api.py::test_resolve_methods tests/vegalite/v5/tests/test_api.py::test_layer_encodings tests/vegalite/v5/tests/test_api.py::test_add_selection tests/vegalite/v5/tests/test_api.py::test_repeat_add_selections tests/vegalite/v5/tests/test_api.py::test_facet_add_selections tests/vegalite/v5/tests/test_api.py::test_layer_add_selection tests/vegalite/v5/tests/test_api.py::test_compound_add_selections[concat] tests/vegalite/v5/tests/test_api.py::test_compound_add_selections[hconcat] tests/vegalite/v5/tests/test_api.py::test_compound_add_selections[vconcat] tests/vegalite/v5/tests/test_api.py::test_selection_property tests/vegalite/v5/tests/test_api.py::test_LookupData tests/vegalite/v5/tests/test_api.py::test_themes tests/vegalite/v5/tests/test_api.py::test_chart_from_dict tests/vegalite/v5/tests/test_api.py::test_consolidate_datasets tests/vegalite/v5/tests/test_api.py::test_consolidate_InlineData tests/vegalite/v5/tests/test_api.py::test_repeat tests/vegalite/v5/tests/test_api.py::test_data_property tests/vegalite/v5/tests/test_api.py::test_subcharts_with_same_data[data.json-layer] tests/vegalite/v5/tests/test_api.py::test_subcharts_with_same_data[data.json-hconcat] tests/vegalite/v5/tests/test_api.py::test_subcharts_with_same_data[data.json-vconcat] tests/vegalite/v5/tests/test_api.py::test_subcharts_with_same_data[data.json-concat] tests/vegalite/v5/tests/test_api.py::test_subcharts_with_same_data[data1-layer] tests/vegalite/v5/tests/test_api.py::test_subcharts_with_same_data[data1-hconcat] tests/vegalite/v5/tests/test_api.py::test_subcharts_with_same_data[data1-vconcat] tests/vegalite/v5/tests/test_api.py::test_subcharts_with_same_data[data1-concat] tests/vegalite/v5/tests/test_api.py::test_subcharts_different_data[data.json-layer] tests/vegalite/v5/tests/test_api.py::test_subcharts_different_data[data.json-hconcat] tests/vegalite/v5/tests/test_api.py::test_subcharts_different_data[data.json-vconcat] tests/vegalite/v5/tests/test_api.py::test_subcharts_different_data[data.json-concat] tests/vegalite/v5/tests/test_api.py::test_subcharts_different_data[data1-layer] tests/vegalite/v5/tests/test_api.py::test_subcharts_different_data[data1-hconcat] tests/vegalite/v5/tests/test_api.py::test_subcharts_different_data[data1-vconcat] tests/vegalite/v5/tests/test_api.py::test_subcharts_different_data[data1-concat] tests/vegalite/v5/tests/test_api.py::test_layer_facet tests/vegalite/v5/tests/test_api.py::test_layer_errors tests/vegalite/v5/tests/test_api.py::test_resolve[layer] tests/vegalite/v5/tests/test_api.py::test_resolve[hconcat] tests/vegalite/v5/tests/test_api.py::test_resolve[vconcat] tests/vegalite/v5/tests/test_api.py::test_resolve[concat] tests/vegalite/v5/tests/test_api.py::test_resolve[facet] tests/vegalite/v5/tests/test_api.py::test_resolve[facet_encoding] tests/vegalite/v5/tests/test_api.py::test_resolve[repeat] tests/vegalite/v5/tests/test_api.py::test_facet[None-chart] tests/vegalite/v5/tests/test_api.py::test_facet[None-layer] tests/vegalite/v5/tests/test_api.py::test_facet[facet-chart] tests/vegalite/v5/tests/test_api.py::test_facet[facet-layer] tests/vegalite/v5/tests/test_api.py::test_facet[row-chart] tests/vegalite/v5/tests/test_api.py::test_facet[row-layer] tests/vegalite/v5/tests/test_api.py::test_facet[column-chart] tests/vegalite/v5/tests/test_api.py::test_facet[column-layer] tests/vegalite/v5/tests/test_api.py::test_sequence tests/vegalite/v5/tests/test_api.py::test_graticule tests/vegalite/v5/tests/test_api.py::test_sphere tests/vegalite/v5/tests/test_api.py::test_validate_dataset