# 什么是在块中迭代列表的最“pythonic”方式？

``for i in xrange(0, len(ints), 4): # dummy op for example code foo += ints[i] * ints[i + 1] + ints[i + 2] * ints[i + 3]` `

` `while ints: foo += ints[0] * ints[1] + ints[2] * ints[3] ints[0:4] = []` `

` `from itertools import izip_longest def grouper(iterable, n, fillvalue=None): args = [iter(iterable)] * n return izip_longest(*args, fillvalue=fillvalue)` `

` `grouper('ABCDEFG', 3, 'x') --> 'ABC' 'DEF' 'Gxx'` `

` `def chunker(seq, size): return (seq[pos:pos + size] for pos in xrange(0, len(seq), size))` `

` `text = "I am a very, very helpful text" for group in chunker(text, 7): print repr(group), # 'I am a ' 'very, v' 'ery hel' 'pful te' 'xt' print '|'.join(chunker(text, 10)) # I am a ver|y, very he|lpful text animals = ['cat', 'dog', 'rabbit', 'duck', 'bird', 'cow', 'gnu', 'fish'] for group in chunker(animals, 3): print group # ['cat', 'dog', 'rabbit'] # ['duck', 'bird', 'cow'] # ['gnu', 'fish']` `

` `chunkSize= 4 for i in xrange(0, len(ints), chunkSize): chunk = ints[i:i+chunkSize] # process chunk of size <= chunkSize` `
` `import itertools def chunks(iterable,size): it = iter(iterable) chunk = tuple(itertools.islice(it,size)) while chunk: yield chunk chunk = tuple(itertools.islice(it,size)) # though this will throw ValueError if the length of ints # isn't a multiple of four: for x1,x2,x3,x4 in chunks(ints,4): foo += x1 + x2 + x3 + x4 for chunk in chunks(ints,4): foo += sum(chunk)` `

` `import itertools def chunks2(iterable,size,filler=None): it = itertools.chain(iterable,itertools.repeat(filler,size-1)) chunk = tuple(itertools.islice(it,size)) while len(chunk) == size: yield chunk chunk = tuple(itertools.islice(it,size)) # x2, x3 and x4 could get the value 0 if the length is not # a multiple of 4. for x1,x2,x3,x4 in chunks2(ints,4,0): foo += x1 + x2 + x3 + x4` `
` `from itertools import izip_longest def chunker(iterable, chunksize, filler): return izip_longest(*[iter(iterable)]*chunksize, fillvalue=filler)` `

` `def chunker(seq, size): res = [] for el in seq: res.append(el) if len(res) == size: yield res res = [] if res: yield res` `

` `>>> list(chunker([i for i in range(10)], 3)) [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]` `

` `>>> list(chunker(set([i for i in range(10)]), 3)) [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]` `

` `>>> list(chunker((i for i in range(10)), 3)) [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]` `

` `>>> def chunker(iterable, chunksize): ... return map(None,*[iter(iterable)]*chunksize)` `

` `>>> s = '1234567890' >>> chunker(s, 3) [('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9'), ('0', None, None)] >>> chunker(s, 4) [('1', '2', '3', '4'), ('5', '6', '7', '8'), ('9', '0', None, None)] >>> chunker(s, 5) [('1', '2', '3', '4', '5'), ('6', '7', '8', '9', '0')]` `

` `it = iter([1, 2, 3, 4, 5, 6, 7, 8, 9]) for chunk in zip(it, it, it, it): print chunk >>> (1, 2, 3, 4) >>> (5, 6, 7, 8)` `

` `>>> def chunker(iterable, chunksize): ... return zip(*[iter(iterable)]*chunksize)` `

` `>>> s = '1234567890' >>> chunker(s, 3) [('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9')] >>> chunker(s, 4) [('1', '2', '3', '4'), ('5', '6', '7', '8')] >>> chunker(s, 5) [('1', '2', '3', '4', '5'), ('6', '7', '8', '9', '0')]` `

` `>>> from itertools import izip >>> def chunker(iterable, chunksize): ... return izip(*[iter(iterable)]*chunksize)` `

` `>>> from itertools import chain, izip, repeat >>> def chunker(iterable, chunksize, fillvalue=None): ... it = chain(iterable, repeat(fillvalue, chunksize-1)) ... args = [it] * chunksize ... return izip(*args)` `

` `data = [...] chunk_size = 10000 # or whatever chunks = [data[i:i+chunk_size] for i in xrange(0,len(data),chunk_size)] for chunk in chunks: ...` `

` `def get_chunk(iterable, chunk_size): result = [] for item in iterable: result.append(item) if len(result) == chunk_size: yield tuple(result) result = [] if len(result) > 0: yield tuple(result) for x in get_chunk([1,2,3,4,5,6,7,8,9,10], 3): print x (1, 2, 3) (4, 5, 6) (7, 8, 9) (10,)` `

` `from itertools import islice def group(it, size): it = iter(it) return iter(lambda: tuple(islice(it, size)), ())` `

` `from itertools import islice, chain, repeat def group_pad(it, size, pad=None): it = chain(iter(it), repeat(pad)) return iter(lambda: tuple(islice(it, size)), (pad,) * size)` `

` `_no_pad = object() def group(it, size, pad=_no_pad): if pad == _no_pad: it = iter(it) sentinel = () else: it = chain(iter(it), repeat(pad)) sentinel = (pad,) * size return iter(lambda: tuple(islice(it, size)), sentinel)` `

` `ints = array([1, 2, 3, 4, 5, 6, 7, 8]) for int1, int2 in ints.reshape(-1, 2): print(int1, int2)` `

` `1 2 3 4 5 6 7 8` `

` `ints = ints[4:]` `

` `def grouper(n, iterable, fillvalue=None): #"grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" args = [iter(iterable)] * n return itertools.izip_longest(fillvalue=fillvalue, *args)` `

` `def grouper(size, iterable): i = iter(iterable) while True: out = [] try: for _ in range(size): out.append(i.next()) except StopIteration: yield out break yield out` `

` `def grouper(size, iterable): it = iter(iterable) while True: group = tuple(itertools.islice(it, None, size)) if not group: break yield group` `

` `def grouper(n, iterable, fillvalue=None): #"grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" args = [iter(iterable)] * n for i in itertools.izip_longest(fillvalue=fillvalue, *args): if tuple(i)[-1] == fillvalue: yield tuple(v for v in i if v != fillvalue) else: yield i` `

1）容易理解
2）适用于任何迭代，而不仅仅是序列（上面的一些答案会扼杀文件句柄）
3）不一次加载块到内存中
4）不会在内存中创build一个相同迭代器的引用列表
5）在列表末尾没有填充值

` `def chunkiter(iterable, size): def inneriter(first, iterator, size): yield first for _ in xrange(size - 1): yield iterator.next() it = iter(iterable) while True: yield inneriter(it.next(), it, size) In [2]: i = chunkiter('abcdefgh', 3) In [3]: for ii in i: for c in ii: print c, print '' ...: abcdefgh` `

1）在外部循环中继续不能按预期工作 – 它只是继续到下一个项目，而不是跳过一个块。 然而，这看起来不是一个问题，因为在外部循环中没有什么可以testing的。
2）break在内部循环中无法正常工作 – 控制将在迭代器中的下一个项目中再次在内部循环中结束。 为了跳过整个块，可以将内部迭代器（ii）置于一个元组中，例如`for c in tuple(ii)` ，或者设置一个标志并且耗尽迭代器。

` `def group_by(iterable, size): """Group an iterable into lists that don't exceed the size given. >>> group_by([1,2,3,4,5], 2) [[1, 2], [3, 4], [5]] """ sublist = [] for index, item in enumerate(iterable): if index > 0 and index % size == 0: yield sublist sublist = [] sublist.append(item) if sublist: yield sublist` `

` `from funcy import partition for a, b, c, d in partition(4, ints): foo += a * b * c * d` `

` `>>> for k, g in itertools.groupby(xrange(35), lambda x: x/10): ... list(g)` `

` `... 0 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] 1 [10, 11, 12, 13, 14, 15, 16, 17, 18, 19] 2 [20, 21, 22, 23, 24, 25, 26, 27, 28, 29] 3 [30, 31, 32, 33, 34] >>>` `

` `for k,g in itertools.groupby(xrange(35), lambda x: x/10): for i in g: # do what you need to do with individual items # now do what you need to do with the whole group` `

` ` messages = a_generator_which_would_not_be_smart_as_a_list for idx, batch in groupby(messages, lambda x: x/1000): batch_request = BatchHttpRequest() for message in batch: batch_request.add(self.service.users().messages().modify(userId='me', id=message['id'], body=msg_labels)) http = httplib2.Http() self.credentials.authorize(http) batch_request.execute(http=http)` `

` `def chunker(iterable, chunksize): return zip(*[iter(iterable)]*chunksize)` `

` `class IteratorExhausted(Exception): pass def translate_StopIteration(iterable, to=IteratorExhausted): for i in iterable: yield i raise to # StopIteration would get ignored because this is generator, # but custom exception can leave the generator. def custom_zip(*iterables, reductor=tuple): iterators = tuple(map(translate_StopIteration, iterables)) while True: try: yield reductor(next(i) for i in iterators) except IteratorExhausted: # when any of iterators get exhausted. break` `

` `def chunker(data, size, reductor=tuple): return custom_zip(*[iter(data)]*size, reductor=reductor)` `

` `>>> for i in chunker('12345', 2): ... print(repr(i)) ... ('1', '2') ('3', '4') >>> for i in chunker('12345', 2, ''.join): ... print(repr(i)) ... '12' '34'` `

` `def chunks(seq, size): it = iter(seq) while True: ret = tuple(it.next() for _ in range(size)) if len(ret) == size: yield ret else: raise StopIteration()` `

` `>>> def foo(): ... i = 0 ... while True: ... i += 1 ... yield i ... >>> c = chunks(foo(), 3) >>> c.next() (1, 2, 3) >>> c.next() (4, 5, 6) >>> list(chunks('abcdefg', 2)) [('a', 'b'), ('c', 'd'), ('e', 'f')]` `
` `def chunker(iterable, n): """Yield iterable in chunk sizes. >>> chunks = chunker('ABCDEF', n=4) >>> chunks.next() ['A', 'B', 'C', 'D'] >>> chunks.next() ['E', 'F'] """ it = iter(iterable) while True: chunk = [] for i in range(n): try: chunk.append(it.next()) except StopIteration: yield chunk raise StopIteration yield chunk if __name__ == '__main__': import doctest doctest.testmod()` `

` `def chunk_iter(iterable, chunk_size): it = iter(iterable) while True: chunk = tuple(next(it) for _ in range(chunk_size)) if not chunk: break yield chunk` `

` `def split_seq(seq, size): newseq = [] splitsize = 1.0/size*len(seq) for i in range(size): newseq.append(seq[int(round(i*splitsize)):int(round((i+1)*splitsize))]) return newseq` `

` `# Four lists of four elements each. l1 = range(0, 4) l2 = range(4, 8) l3 = range(8, 12) l4 = range(12, 16) for i1, i2, i3, i4 in zip(l1, l2, l3, l4): ...` `

` `>>> print l1 [0, 1, 2, 3] >>> print l2 [4, 5, 6, 7] >>> print l3 [8, 9, 10, 11] >>> print l4 [12, 13, 14, 15] >>> print zip(l1, l2, l3, l4) [(0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)]` `

` `from itertools import izip for i1, i2, i3, i4 in izip(l1, l2, l3, l4): ...` `

` `for a, b, c, d in zip(x[0::4], x[1::4], x[2::4], x[3::4]): ... do something with a, b, c and d ...` `

` `def chunker(iterable, size, reductor, condition): it = iter(iterable) def chunk_generator(): return (next(it) for _ in range(size)) chunk = reductor(chunk_generator()) while condition(chunk): yield chunk chunk = reductor(chunk_generator())` `

# 参数：

### 明显的

• `iterable`是任何迭代/迭代器/发生器包含/产生/迭代input数据，
• `size`当然是你想要的块的大小，

### 更有意思的

• `reductor`是可调用的，它接收发生器遍历chunk的内容。
我希望它会返回序列或string，但我不要求。

你可以通过这个参数来传递例如`list``tuple``set``frozenset`
或更奇特的。 我会通过这个函数，返回string
（前提是`iterable`包含/生成/迭代string）：

` `def concatenate(iterable): return ''.join(iterable)` `

请注意， `reductor`可以通过引发exception来引起闭合发生器。

• `condition`是什么`reductor`返回接收任何东西的callable。
它决定批准和产生（通过返回任何评估为`True` ），
或拒绝它并完成发电机的工作（通过返回其他任何东西或引发exception）。

`iterable`的元素的数量不能被`size`整除时，当`it`被耗尽时， `reductor`将接收到发生器产生的元素less于`size`
我们称这些元素为元素

我邀请了两个函数作为这个parameter passing：

• `lambda x:x`最后的元素将被放弃。

• `lambda x: len(x)==<size>`最后的元素将被拒绝。
用等于`size`数字replace`<size>`

` `groupby(iterable, (lambda x,y: (lambda z: x.next()/y))(count(),100))` `

` `for k,v in groupby(bigdata, (lambda x,y: (lambda z: x.next()/y))(count(),100))): cursor.executemany(sql, v)` `

` `import itertools def split_groups(iter_in, group_size): return ((x for _, x in item) for _, item in itertools.groupby(enumerate(iter_in), key=lambda x: x[0] // group_size)) for x, y, z, w in split_groups(range(16), 4): foo += x * y + z * w` `

` `def split(what, target_length=79): '''splits list of strings into sublists, each having string length at most 79''' out = [[]] while what: if len("', '".join(out[-1])) + len(what[0]) < target_length: out[-1].append(what.pop(0)) else: if not out[-1]: # string longer than target_length out[-1] = [what.pop(0)] out.append([]) return out` `

` `>>> split(['deferred_income', 'long_term_incentive', 'restricted_stock_deferred', 'shared_receipt_with_poi', 'loan_advances', 'from_messages', 'other', 'director_fees', 'bonus', 'total_stock_value', 'from_poi_to_this_person', 'from_this_person_to_poi', 'restricted_stock', 'salary', 'total_payments', 'exercised_stock_options'], 75) [['deferred_income', 'long_term_incentive', 'restricted_stock_deferred'], ['shared_receipt_with_poi', 'loan_advances', 'from_messages', 'other'], ['director_fees', 'bonus', 'total_stock_value', 'from_poi_to_this_person'], ['from_this_person_to_poi', 'restricted_stock', 'salary', 'total_payments'], ['exercised_stock_options']]` `