Generators 路 itertools 路 import system

By on

馃悕 Python: Generators, itertools, import + modules

SPbPU, Viktor Tiulpin, 2021


In the previous episodes

  • basic stuff, syntax
  • functions (wrote our cool min function) + functional stuff
  • scopes, PEP-8
  • strings, bytes
  • collections
  • classes basics
  • decorators, functools
  • exceptions, context managers
  • iterators

馃懟 Generators


馃憞 Generator

>>> def g():
        print("Starting")
        x = 42
        yield x
        x += 1
        yield x
        print("Done")
>>> type(g)
<class 'function'>
>>> gen = g()
>>> type(gen)
<class 'generator'>
>>> next(gen)
Starting    
42
>>> next(gen)
43

after yield it stops and comes back later


unique

>>> def unique(iterable, seen=None):
        seen = set(seen or [])
        for item in iterable:
            if item not in seen:
                seen.add(item)
                yield item
   
>>> xs = [1, 1, 2, 3]
>>> unique(xs)
<generator object unique at 0x1027c5798>
>>> list(unique(xs))
[1, 2, 3]
>>> 1 in unique(xs)
True

map

>>> def map(func, iterable, *rest):
        for args in zip(iterable, *rest):
            yield func(*args)
   
>>> xs = range(5)
>>> map(lambda x: x * x, xs)
<generator object map at 0x103122510>
>>> list(map(lambda x: x * x, xs))
[0, 1, 4, 9, 16]
>>> 9 in map(lambda x: x * x, xs)
True

chain

>>> def chain(*iterables):
        for iterable in iterables:
            for item in iterable:
                yield item
   
>>> xs = range(3)
>>> ys = [42]
>>> chain(xs, ys)
<generator object chain at 0x10311d708>
>>> list(chain(xs, ys))
[0, 1, 2, 3, 42]
>>> 42 in chain(xs, ys)
True

count

>>> def count(start=0):
        while True:
            yield start
            start += 1
   
>>> next(count())
0
>>> counter = count()
>>> next(counter)
0
>>> next(counter)
1

How to reuse: don鈥檛.

>>> def g():
    yield 42
   
>>> gen = g()
>>> list(gen)
[42]
>>> list(gen) # WASTED
[]

or use tee from 矛tertools


yield

>>> def g():
        res = yield # entrypoint 1 # ???
        print("Got {!r}".format(res))
        res = yield 42 # entrypoint 2
        print("Got {!r}".format(res))
   
>>> gen = g()
>>> next(gen) # until first yield
>>> next(gen) # until second yield
Got 'None'
42
>>> next(gen) # until the end
Got 'None'
Traceback (most recent call last):
    File "<stdin>", line 1, in <module>
StopIteration

send!

>>> gen = g()
>>> gen.send("foobar")
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: can't send [   ] to a just-started generator
>>> gen = g()
>>> gen.send(None) # 鈮 next(gen)
>>> gen.send("foobar")
Got 'foobar'
42

throw

>>> def g():
        try:
            yield 42
        except ValueError as e:
            yield e
        finally:
            print("Done")
   
>>> gen = g()
>>> next(gen)
42
>>> gen.throw(ValueError, "something is wrong")
ValueError('something is wrong',)
>>> gen = g()
>>> next(gen)
42
>>> gen.close()
Done

and close()


yield from and return

>>> def f():
        yield 42
        return []
   
>>> def g():
        res = yield from f()
        print("Got {!r}".format(res))
   
>>> gen = g()
>>> next(gen)
42
>>> next(gen, None)
Got []

generators and context managers

import tempfile
import shutil

@contextmanager
def tempdir(): # __init__
    outdir = tempfile.mkdtemp() # __enter__
    try:
        yield outdir # ---------
    finally:
        shutil.rmtree(outdir) # __exit__

with tempdir() as path:
    print(path) 

useful example?


Generators wrap-up

  • generator is a function which use yield and yield from
  • generators are everywhere
  • generators can be used as iterators, coroutines, easy context managers and others!

馃敤 itertools


islice

>>> from itertools import islice
>>> xs = range(10)
>>> list(islice(xs, 3)) # 鈮 xs[:3]
[0, 1, 2]
>>> list(islice(xs, 3, None)) # 鈮 xs[3:]
[3, 4, 5, 6, 7, 8, 9]
>>> list(islice(xs, 3, 8, 2)) # 鈮 xs[3:8:2]
[3, 5, 7]

count, cycle, repeat

>>> def take(n, iterable):
        return list(islice(iterable, n))
   
>>> list(take(range(10), 3))
[0, 1 2]
>>> from itertools import count, cycle, repeat
>>> take(3, count(0, 5))
[0, 5, 10]
>>> take(3, cycle([1, 2, 3]))
[1, 2, 3]
>>> take(3, repeat(42))
[42, 42, 42]
>>> take(3, repeat(42, 2)) # 2 is number of repeats here
[42, 42]

chain

>>> from itertools import chain
>>> take(5, chain(range(2), range(5, 10)))
[0, 1, 5, 6, 7]
>>> it = (range(x, x ** x) for x in range(2, 4))
>>> take(5, chain.from_iterable(it))
[2, 3, 3, 4, 5]

chain.from_iterable(it) vs chain(*it)?


tee

>>> from itertools import tee
>>> it = range(3)
>>> a, b, c = tee(it, 3)
>>> list(a), list(b), list(c)
([0, 1, 2], [0, 1, 2], [0, 1, 2])

don鈥檛 use it after copying


combinatorics!

>>> list(itertools.product("AB", repeat=2))
[('A', 'A'), ('A', 'B'), ('B', 'A'), ('B', 'B')]
>>> list(itertools.product("AB", repeat=3))
[('A', 'A', 'A'), ('A', 'A', 'B'), ('A', 'B', 'A'),    ]
>>> list(itertools.permutations("AB"))
[('A', 'B'), ('B', 'A')]
>>> from itertools import combinations, \
    combinations_with_replacement
>>> list(combinations("ABC", 2))
[('A', 'B'), ('A', 'C'), ('B', 'C')]
>>> list(combinations_with_replacement("ABC", 2))
[('A', 'A'), ('A', 'B'), ('A', 'C'),
('B', 'B'), ('B', 'C'), ('C', 'C')]

itertools wrap-up

  • useful
  • combinatorics,
  • islice,
  • count, cycle, repeat
  • tee

鉃★笍 Modules, imports and other stuff


http://www.dabeaz.com/modulepackage/


1锔忊儯 Modules


module.py

"""I'm a module."""

some_variable = "variable"


def foo():
    return 42

with tempfile

>>> import useful
>>> dir(module)
[   , '__cached__', '__doc__', '__file__', '__name__',
'foo', 'some_variable']

python ./module.py

def test():
    assert boo() == 4


if __name__ == "__main__":
    print("Running tests     ")
    test()
    print("OK")

import

>>> import module # 懈褋锌芯谢薪褟械褌 屑芯写褍谢褜 褋胁械褉褏褍 胁薪懈蟹
>>> module
<module 'module' from './module.py'>
>>> import module as alias
>>> alias
<module 'module' from './module.py'>
>>> from useful import foo as boo, some_variable
>>> foo()
42
>>> some_variable
'variable'

sys.path

>>> import sys
>>> sys.path
['', '/usr/local/lib/python3.8',    ]

Modules wrap-up

  • module is a *.py file from which you cam import from
  • import from, import as
  • byte code is executed from up to down
  • sort imports
from collections import OrderedDict
from itertools import islice
import os
import sys

2锔忊儯 Packages


package package

package
鈹溾攢鈹 __init__.py # !
鈹溾攢鈹 bar.py
鈹斺攢鈹 foo.py
>>> import package
>>> package
<module 'package' from './package/__init__.py'>
>>> package.foo
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
AttributeError: 'module' object has no attribute 'foo'
>>> package.bar
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
AttributeError: 'module' object has no attribute 'bar'
>>> from package import bar
>>> bar
<module 'package.bar' from './package/bar.py'>

absolute / relative import

import package.foo
from package import bar
from . import foo, bar  # don't, personal opinion :)

executable package

# package/__init__.py
print("package.__init__")

# package/__main__.py
print("It works!")

Now we can run it!

$ python -m package
package.__init__ # ?
It works!

Packages wrap-up

  • package is a way to group Python code
  • every directory with __init__.py makes a package
  • use absolute imports
  • adding __main__.py allows you to execute your package

馃弫 Any questions?