diff --git a/geopandas/tools/sjoin.py b/geopandas/tools/sjoin.py
index 9ac00d7..20bbbc7 100644
--- a/geopandas/tools/sjoin.py
+++ b/geopandas/tools/sjoin.py
@@ -196,6 +196,9 @@ def _geom_predicate_query(left_df, right_df, predicate, distance):
         # within is implemented as the inverse of contains
         # flip back the results
         r_idx, l_idx = l_idx, r_idx
+        indexer = np.lexsort((r_idx, l_idx))
+        l_idx = l_idx[indexer]
+        r_idx = r_idx[indexer]
 
     return l_idx, r_idx
 
@@ -319,14 +322,6 @@ def _adjust_indexers(indices, distances, original_length, how, predicate):
     """
     # the indices represent an inner join, no adjustment needed
     if how == "inner":
-        if predicate == "within":
-            # except for the within predicate, where we switched to contains
-            # with swapped left/right -> need to re-sort to have consistent result
-            l_idx, r_idx = indices
-            indexer = np.lexsort((r_idx, l_idx))
-            indices = l_idx[indexer], r_idx[indexer]
-            if distances is not None:
-                distances = distances[indexer]
         return indices, distances
 
     l_idx, r_idx = indices

diff --git a/geopandas/tools/tests/test_sjoin.py b/geopandas/tools/tests/test_sjoin.py
index 59419fc..b6d0034 100644
--- a/geopandas/tools/tests/test_sjoin.py
+++ b/geopandas/tools/tests/test_sjoin.py
@@ -5,14 +5,21 @@ import numpy as np
 import pandas as pd
 import shapely
 
-from shapely.geometry import Point, Polygon, GeometryCollection
+from shapely.geometry import Point, Polygon, GeometryCollection, box
 
 import geopandas
 import geopandas._compat as compat
-from geopandas import GeoDataFrame, GeoSeries, read_file, sjoin, sjoin_nearest
+from geopandas import (
+    GeoDataFrame,
+    GeoSeries,
+    read_file,
+    sjoin,
+    sjoin_nearest,
+    points_from_xy,
+)
 from geopandas.testing import assert_geodataframe_equal, assert_geoseries_equal
 
-from pandas.testing import assert_frame_equal, assert_series_equal
+from pandas.testing import assert_frame_equal, assert_series_equal, assert_index_equal
 import pytest
 
 
@@ -384,6 +391,39 @@ class TestSpatialJoin:
         joined = sjoin(left, right, how=how, predicate="dwithin", distance=distance)
         assert_frame_equal(expected_gdf.sort_index(), joined.sort_index())
 
+    # GH3239
+    @pytest.mark.parametrize(
+        "predicate",
+        [
+            "contains",
+            "contains_properly",
+            "covered_by",
+            "covers",
+            "crosses",
+            "intersects",
+            "touches",
+            "within",
+        ],
+    )
+    def test_sjoin_left_order(self, predicate):
+        # a set of points in random order -> that order should be preserved
+        # with a left join
+        pts = GeoDataFrame(
+            geometry=points_from_xy([0.1, 0.4, 0.3, 0.7], [0.8, 0.6, 0.9, 0.1])
+        )
+        polys = GeoDataFrame(
+            {"id": [1, 2, 3, 4]},
+            geometry=[
+                box(0, 0, 0.5, 0.5),
+                box(0, 0.5, 0.5, 1),
+                box(0.5, 0, 1, 0.5),
+                box(0.5, 0.5, 1, 1),
+            ],
+        )
+
+        joined = sjoin(pts, polys, predicate=predicate, how="left")
+        assert_index_equal(joined.index, pts.index)
+
 
 class TestIndexNames:
     @pytest.mark.parametrize("how", ["inner", "left", "right"])

REGR: incorrect order of left sjoin with within predicate
I think that #2353 has caused a regression when doing left join with within predicate:

```py
pts = gpd.GeoDataFrame(geometry=gpd.points_from_xy(*np.random.rand(2, 10)))
polys = gpd.GeoDataFrame({"id":[1, 2, 3, 4]}, geometry=[
    box(0, 0, .5, .5),
    box(0, .5, .5, 1),
    box(.5, 0, 1, .5),
    box(.5, .5, 1, 1)
])

pts.sjoin(polys, predicate="within", how="left")

                  geometry  index_right  id
9  POINT (0.12418 0.11896)            0   1
1  POINT (0.22954 0.40218)            0   1
5  POINT (0.42634 0.49957)            0   1
8  POINT (0.34231 0.94855)            1   2
4  POINT (0.93312 0.02055)            2   3
2  POINT (0.99042 0.32572)            2   3
3   POINT (0.6554 0.69657)            3   4
6  POINT (0.82468 0.79515)            3   4
7  POINT (0.59083 0.88989)            3   4
0  POINT (0.79659 0.89147)            3   4

```

The join is correct but the order of rows is not. Any other combination I tried seems okay.

The result is apparently sorted by the index of right rather than the index of left, so it seems that the swap of left/right we do for `"within"` is incorrect at some point.

We should try to fix this before 1.0-alpha1.
geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_left_order[within]
geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_suffixes[left-left-right-expected_cols0] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_suffixes[inner-left-right-expected_cols1] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_suffixes[right-left-right-expected_cols2] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_suffixes[left-lft-rgt-expected_cols3] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_suffixes[inner-lft-rgt-expected_cols4] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_suffixes[right-lft-rgt-expected_cols5] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_crs_mismatch[default-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_crs_mismatch[string-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_unknown_kwargs[default-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[intersects-default-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[intersects-string-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[intersects-named-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[intersects-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[intersects-named-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[contains-default-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[contains-string-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[contains-named-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[contains-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[contains-named-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[within-default-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[within-string-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[within-named-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[within-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_inner[within-named-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[intersects-default-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[intersects-string-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[intersects-named-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[intersects-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[intersects-named-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[contains-default-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[contains-string-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[contains-named-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[contains-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[contains-named-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[within-default-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[within-string-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[within-named-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[within-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_left[within-named-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_empty_join geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty0-contains] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty0-contains_properly] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty0-covered_by] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty0-covers] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty0-crosses] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty0-intersects] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty0-touches] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty0-within] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty1-contains] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty1-contains_properly] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty1-covered_by] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty1-covers] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty1-crosses] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty1-intersects] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty1-touches] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_join_with_empty[empty1-within] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_invalid_args[default-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_invalid_args[string-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[intersects-default-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[intersects-string-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[intersects-named-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[intersects-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[intersects-named-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[contains-default-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[contains-string-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[contains-named-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[contains-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[contains-named-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[within-default-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[within-string-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[within-named-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[within-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_right[within-named-multi-index] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_dwithin[geo_left0-geo_right0-expected_left0-expected_right0-1.4142135623730951-inner] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_dwithin[geo_left1-geo_right1-expected_left1-expected_right1-1.4142135623730951-inner] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_dwithin[geo_left2-geo_right2-expected_left2-expected_right2-distance2-inner] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_dwithin[geo_left3-geo_right3-expected_left3-expected_right3-distance3-inner] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_left_order[contains] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_left_order[contains_properly] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_left_order[covered_by] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_left_order[covers] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_left_order[crosses] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_left_order[intersects] geopandas/tools/tests/test_sjoin.py::TestSpatialJoin::test_sjoin_left_order[touches] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_preserve_index_names[inner] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_preserve_index_names[left] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_preserve_index_names[right] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_preserve_index_names_multiindex[inner] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_preserve_index_names_multiindex[left] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_preserve_index_names_multiindex[right] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_duplicate_column_index_name[inner] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_duplicate_column_index_name[left] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_duplicate_column_index_name[right] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_duplicate_column_index_name_multiindex[inner] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_duplicate_column_index_name_multiindex[left] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_duplicate_column_index_name_multiindex[right] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_conflicting_column_index_name[inner] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_conflicting_column_index_name[left] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_conflicting_column_index_name[right] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_conflicting_column_with_suffix[inner] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_conflicting_column_with_suffix[left] geopandas/tools/tests/test_sjoin.py::TestIndexNames::test_conflicting_column_with_suffix[right] geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_geometry_name geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_left geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_right geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_inner geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_predicate geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_bad_predicate geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_duplicate_column_name geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_named_index[left] geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_named_index[right] geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_named_index[inner] geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_values geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_empty_geometries geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_no_valid_geoms[intersects] geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_no_valid_geoms[within] geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_sjoin_no_valid_geoms[contains] geopandas/tools/tests/test_sjoin.py::TestSpatialJoinNYBB::test_empty_sjoin_return_duplicated_columns geopandas/tools/tests/test_sjoin.py::test_sjoin_inner geopandas/tools/tests/test_sjoin.py::TestNearest::test_allowed_hows[how_kwargs0] geopandas/tools/tests/test_sjoin.py::TestNearest::test_allowed_hows[how_kwargs1] geopandas/tools/tests/test_sjoin.py::TestNearest::test_allowed_hows[how_kwargs2] geopandas/tools/tests/test_sjoin.py::TestNearest::test_allowed_hows[how_kwargs3] geopandas/tools/tests/test_sjoin.py::TestNearest::test_invalid_hows[outer] geopandas/tools/tests/test_sjoin.py::TestNearest::test_invalid_hows[abcde] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_right_df_how_left[None] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_right_df_how_left[distance] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_right_df_how_right[None] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_right_df_how_right[distance] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_left_df[None-inner] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_left_df[None-left] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_left_df[distance-inner] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_left_df[distance-left] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_left_df_how_right[None] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_left_df_how_right[distance] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_join_due_to_max_distance[inner] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_join_due_to_max_distance[left] geopandas/tools/tests/test_sjoin.py::TestNearest::test_empty_join_due_to_max_distance_how_right geopandas/tools/tests/test_sjoin.py::TestNearest::test_max_distance[inner] geopandas/tools/tests/test_sjoin.py::TestNearest::test_max_distance[left] geopandas/tools/tests/test_sjoin.py::TestNearest::test_max_distance_how_right geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left0-geo_right0-expected_left0-expected_right0-distances0-inner] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left0-geo_right0-expected_left0-expected_right0-distances0-left] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left1-geo_right1-expected_left1-expected_right1-distances1-inner] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left1-geo_right1-expected_left1-expected_right1-distances1-left] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left2-geo_right2-expected_left2-expected_right2-distances2-inner] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left2-geo_right2-expected_left2-expected_right2-distances2-left] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left3-geo_right3-expected_left3-expected_right3-distances3-inner] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left3-geo_right3-expected_left3-expected_right3-distances3-left] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left4-geo_right4-expected_left4-expected_right4-distances4-inner] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left4-geo_right4-expected_left4-expected_right4-distances4-left] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left5-geo_right5-expected_left5-expected_right5-distances5-inner] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left5-geo_right5-expected_left5-expected_right5-distances5-left] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left6-geo_right6-expected_left6-expected_right6-distances6-inner] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left6-geo_right6-expected_left6-expected_right6-distances6-left] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left7-geo_right7-expected_left7-expected_right7-distances7-inner] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_left[geo_left7-geo_right7-expected_left7-expected_right7-distances7-left] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_right[geo_left0-geo_right0-expected_left0-expected_right0-distances0] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_right[geo_left1-geo_right1-expected_left1-expected_right1-distances1] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_right[geo_left2-geo_right2-expected_left2-expected_right2-distances2] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_right[geo_left3-geo_right3-expected_left3-expected_right3-distances3] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_right[geo_left4-geo_right4-expected_left4-expected_right4-distances4] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_right[geo_left5-geo_right5-expected_left5-expected_right5-distances5] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_right[geo_left6-geo_right6-expected_left6-expected_right6-distances6] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_right[geo_left7-geo_right7-expected_left7-expected_right7-distances7] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_inner geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_exclusive[None-expected0] geopandas/tools/tests/test_sjoin.py::TestNearest::test_sjoin_nearest_exclusive[1.1-expected1]