用Spark做key排序

#直接调用sortByKey()函数就可以做到
from pyspark import SparkContext

sc = SparkContext('local','Sort')
list = ["7","4","8","2","5"]
textFile = sc.parallelize(list)
textRDD =textFile.map(lambda word : (word, 1))
textRDD.sortByKey().sortByKey().foreach(print)

sortByKey:是对键值进行排序

 def sortByKey(self, ascending=True, numPartitions=None, keyfunc=lambda x: x):
 """
 Sorts this RDD, which is assumed to consist of (key, value) pairs.
 # noqa

        >>> tmp = [('a', 1), ('b', 2), ('1', 3), ('d', 4), ('2', 5)]
        >>> sc.parallelize(tmp).sortByKey().first()
        ('1', 3)
        >>> sc.parallelize(tmp).sortByKey(True, 1).collect()
        [('1', 3), ('2', 5), ('a', 1), ('b', 2), ('d', 4)]
        >>> sc.parallelize(tmp).sortByKey(True, 2).collect()
        [('1', 3), ('2', 5), ('a', 1), ('b', 2), ('d', 4)]
        >>> tmp2 = [('Mary', 1), ('had', 2), ('a', 3), ('little', 4), ('lamb', 5)]
        >>> tmp2.extend([('whose', 6), ('fleece', 7), ('was', 8), ('white', 9)])
        >>> sc.parallelize(tmp2).sortByKey(True, 3, keyfunc=lambda k: k.lower()).collect()
        [('a', 3), ('fleece', 7), ('had', 2), ('lamb', 5),...('white', 9), ('whose', 6)]
        """
        if numPartitions is None:
            numPartitions = self._defaultReducePartitions()

        memory = self._memory_limit()
        serializer = self._jrdd_deserializer

sortBy: 是自定义排序

def sortBy(self, keyfunc, ascending=True, numPartitions=None):
 """
 Sorts this RDD by the given keyfunc

        >>> tmp = [('a', 1), ('b', 2), ('1', 3), ('d', 4), ('2', 5)]
        >>> sc.parallelize(tmp).sortBy(lambda x: x[0]).collect()
        [('1', 3), ('2', 5), ('a', 1), ('b', 2), ('d', 4)]
        >>> sc.parallelize(tmp).sortBy(lambda x: x[1]).collect()
        [('a', 1), ('b', 2), ('1', 3), ('d', 4), ('2', 5)]
        """
        return self.keyBy(keyfunc).sortByKey(ascending, numPartitions).values()

问题:当创建的RDd是从本地读取时，sortByKey排序出错

实验数据:

代码:

　　from pyspark import SparkContext

sc = SparkContext('local','Sort')
textFile = sc.textFile("file:///usr/local/spark/mycode/TestPackage/sort.txt")
textRDD =textFile.flatMap(lambda line : line.split(" ")).map(lambda word : (word, 1))
textRDD.sortByKey().foreach(print)
 
效果:
('18', 1)
('2', 1)
('34', 1)
('34', 1)
('34', 1)
('67', 1)
('89', 1)
('892', 1)
 