Search query with aggregation and sorting

in the below mentioned elasticsearch search query, I want to fetch the documents in a way that all have different variant ids and the product which is fetched from the variant group has the least sort order.

{
    "from": 0,
    "size": 40,
    "query": {
        "bool": {
            "must": [
                {
                    "match": {
                        "card_ids": 519991
                    }
                },
                {
                    "term": {
                        "is_parent": true
                    }
                },
                {
                    "term": {
                        "flag_hidden": false
                    }
                }
            ]
        }
    },
    "aggs": {
        "variant_groups": {
            "terms": {
                "field": "variant_info.variant_id",
                "size": 40
            },
            "aggs": {
                "min_sort_order": {
                    "min": {
                        "field": "variant_info.sort_order"
                    }
                },
                "top_hits": {
                    "top_hits": {
                        "size": 1,
                        "sort": [
                            {
                                "variant_info.sort_order": "asc"
                            }
                        ],
                        "_source": [
                            "variant_info",
                            "mrp",
                            "pid",
                            "name"
                        ]
                    }
                }
            }
        }
    },
    "sort": [
        {
            "mrp": "asc",
            "card_id_map.519991.position": "asc"
        }
    ],
    "_source": [
        "pid",
        "name",
        "mrp"
    ]
}

this is my variant info object within the document

"variant_info": [
    {
        "variant_id": 6731,
        "sort_order": 2
    }
]

the order of products which i am getting in the aggregated buckets in the resposne is not the according to the sorting that i have defined which is mrp and card_id_map.519991.position