feat: add funcy and some update

Change-Id: I29e5e1b012a122e4244e90d2a4582430946edff8

Author: windard

README.md

@@ -315,6 +315,12 @@ python的强大之处有很大的一方面在于它有各种各样非常强大
 
 ## [tqdm](content/tqdm.md)
 
+## [schematics](content/schematics.md)
+
+## [pyautogui](content/pyautogui.md)
+
+## [funcy](content/funcy.md)
+
 ## [tools](content/tools.md)
 
 ## [Other_thing](content/other_thing.md)

code/funcy_cache_demo.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+import time
+import funcy as fc
+
+from functools import wraps
+
+
+def cache(func):
+    local_cache = {}
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        key = "{}:{}".format(";".join(map(str, args)),
+                             ";".join(["{}:{}".format(*map(str, item)) for item in sorted(kwargs.items())]))
+        if key in local_cache:
+            return local_cache[key]
+        else:
+            value = func(*args, **kwargs)
+            local_cache[key] = value
+            return value
+
+    return wrapper
+
+
+# @cache
+@fc.memoize
+def get_timestamp(x):
+    return int(time.time())
+
+
+if __name__ == '__main__':
+    print(get_timestamp(1))
+    time.sleep(1)
+    print(get_timestamp(1))
+    time.sleep(1)
+    print(get_timestamp(1))
+    time.sleep(2)
+    print(get_timestamp(2))
+    time.sleep(1)
+    print(get_timestamp(2))
+    print(get_timestamp.memory)

code/funcy_demo.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+import time
+import funcy as fc
+
+from functools import wraps
+from compiler.ast import flatten as ast_flatten
+
+
+def flatten(elements):
+    return [_ for item in elements for _ in flatten(item)] if isinstance(elements, list) else [elements]
+
+
+@fc.once
+def call_once():
+    print("only once called")
+
+
+@fc.once
+def call_once_with_args(x):
+    print("only once with args called")
+
+
+if __name__ == '__main__':
+    a = [1, 2, [3, 4, [5, 6], [7, 8]], [9, 10]]
+    print(fc.flatten(a))
+    print(ast_flatten(a))
+    print(flatten(a))
+
+    call_once()
+    call_once()
+    call_once_with_args(1)
+    call_once_with_args(2)

code/funcy_flatten_demo.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+
+import funcy as fc
+# only support in python2
+from compiler.ast import flatten as ast_flatten
+
+
+def flatten(elements):
+    result = []
+    for item in elements:
+        if isinstance(item, list):
+            result.extend(flatten(item))
+        else:
+            result.append(item)
+    return result
+
+
+def old_flatten(elements):
+    return [_ for item in elements for _ in old_flatten(item)] if isinstance(elements, list) else [elements]
+    # result = []
+    # for item in elements:
+    #     if isinstance(item, list):
+    #         item = old_flatten(item)
+    #     else:
+    #         item = [item]
+    #     result.extend(item)
+    # return result
+
+
+if __name__ == '__main__':
+    a = [1, 2, [3, 4, [5, 6], [7, 8]], [9, 10]]
+    print(fc.flatten(a))
+    print(ast_flatten(a))
+    print(flatten(a))

code/funcy_list_demo.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+import funcy as fc
+
+if __name__ == '__main__':
+    a = [1,3,4,5,2,5,1,5,2,5,1]
+    print(fc.distinct(a))
+    print(set(a))
+
+    # 按个数分组,舍弃多余的元素
+    print(fc.partition(2, range(10)))
+    print(fc.partition(3, range(10)))
+    # 按个数分组,多余的元素单列
+    print(fc.chunks(2, range(10)))
+    print(fc.chunks(3, range(10)))
+
+    # 此处不能用 lstrip 或者 rstrip, 因为会将输入字符串当成字符数组
+    print("open_api_enforce_interface".lstrip("open_api"))
+    print("open_api_enforce_interface".rstrip("_interface"))
+    print(fc.cut_prefix("open_api_test_interface", "open_api"))
+    print(fc.cut_suffix("open_api_test_interface", "interface"))

code/funcy_once_demo.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+import funcy as fc
+
+
+@fc.once
+def call_once():
+    print("only once called")
+
+
+@fc.once
+def call_once_with_args(x):
+    print("only once with args called")
+
+
+if __name__ == '__main__':
+    call_once()
+    call_once()
+    call_once_with_args(1)
+    call_once_with_args(2)

code/funcy_retry_demo.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+import time
+import funcy as fc
+
+from functools import wraps
+
+# def retry(times=3):
+#     def inner(func):
+#         @wraps(func)
+#         def wrapper(*args, **kwargs):
+#             for i in range(times):
+#                 try:
+#                     return func(*args, **kwargs)
+#                 except Exception as e:
+#                     if i == times-1:
+#                         raise e
+#                     print("occur error:%r" % e)
+#
+#         return wrapper
+#     return inner
+
+# def retry(times=3):
+#     def inner(func):
+#         @wraps(func)
+#         def wrapper(*args, **kwargs):
+#             try:
+#                 return func(*args, **kwargs)
+#             except Exception as e:
+#                 if times <= 1:
+#                     raise e
+#                 print("occur error:%r" % e)
+#                 return retry(times-1)(func)(*args, **kwargs)
+#
+#         return wrapper
+#
+#     return inner
+
+
+def retry(times=3):
+    def inner(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            count = 0
+            while count < times:
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    print("occur error:%r" % e)
+                    count += 1
+            raise
+        return wrapper
+
+    return inner
+
+
+@fc.retry(5)
+# @retry(3)
+def raise_exception():
+    time.sleep(1)
+    return 1 / 0
+
+
+if __name__ == '__main__':
+    print(raise_exception())

content/concurrent.md

@@ -4,8 +4,27 @@ python 中的异步库，集成 threading 和 multiprocessing 两个库。
 
 可以使用 `ThreadPoolExecutor` 和 `ProcessPoolExecutor` 来做线程和进程
 
-
-```
+### 简单使用
+
+这个库使用很简单，常见方法也即那么几种
+> [concurrent.futures --- 启动并行任务](https://docs.python.org/zh-cn/3/library/concurrent.futures.html)
+
+- concurrent.futures.Executor|ThreadPoolExecutor|ProcessPoolExecutor
+    - submit
+    - map
+    - shutdown
+- concurrent.futures.Future
+    - cancel
+    - cancel
+    - cancel
+    - done
+    - cancel
+    - exception
+    - add_done_callback
+- concurrent.futures.wait
+- concurrent.futures.as_completed
+
+```python
 # -*- coding: utf-8 -*-
 
 import time
@@ -76,12 +95,78 @@ if __name__ == '__main__':
 
 ```
 
-参考链接
+### 动态添加任务
+
+```python
+# -*- coding: utf-8 -*-
+import time
+import random
+import traceback
 
-[使用Python进行并发编程-PoolExecutor篇](http://www.dongwm.com/archives/%E4%BD%BF%E7%94%A8Python%E8%BF%9B%E8%A1%8C%E5%B9%B6%E5%8F%91%E7%BC%96%E7%A8%8B-PoolExecutor%E7%AF%87/)
+from concurrent import futures
+from concurrent.futures import ThreadPoolExecutor
+
+result = []
+
+
+def handle(num, pool):
+    try:
+        print("thread for:%d" % num)
+        sleep_time = random.random()
+        time.sleep(sleep_time)
+        if sleep_time < 0.95:
+            print("thread pool add one")
+            print("before pool size:%d" % pool._work_queue.qsize())
+            pool.submit(handle, num*10, pool)
+            # result.append(pool.submit(handle, 111, pool))
+            print("after pool size:%d" % pool._work_queue.qsize())
+    except Exception as e:
+        # 问题在这里:
+        # py2: cannot schedule new futures after shutdown
+        # py3: cannot schedule new futures after interpreter shutdown
+        print(repr(e))
+        traceback.print_stack()
+
+
+if __name__ == '__main__':
+    # 使用 map 的时候需要注意，其实 pool 会自动帮你把参数 zip 一下，先合并再分别映射
+    # 其实最后不用调用 shutdown 也会自动等待所有任务结束，然后主进程结束
+    thread_pool = ThreadPoolExecutor(30)
+    # map_result = thread_pool.map(handle, range(2), [thread_pool] * 2)
+    # result.extend(list(map_result))
+    # 实际上，使用 submit 多个参数的时候也需要注意，参数直接摆开，和 threading 不一样
+    for i in range(4):
+        result.append(thread_pool.submit(handle, i, thread_pool))
+    print("start threading")
+    # 在没有 shutdown 之后为什么还会自动结束呢？谁在
+    # thread_pool.shutdown()
+    while result:
+        print("before result length:%d" % len(result))
+        for f in futures.as_completed(result):
+            result.remove(f)
+        # 此处还是有风险，如果这里结束了，全都结束了之后，还是会自动调用 shutdown
+        # 其实还有待提交的请求，这样就尴尬了。
+        # 但是加等待时间也不能够
+        if len(result) == 0:
+            time.sleep(2)
+        print("after result length:%d" % len(result))
+        # 还是用 lock 锁定一个计数器比较现实，计数器归零再结束。
+
+    # futures.wait(result)
+    # thread_pool.shutdown()
+    # 不需要 as_completed 也不需要手动 wait，实际上在 shutdown 的时候其实就是会等待结束的
+    # 如果使用上下文管理器的话会自动 shutdown
+    # 问题是在子线程中不能再往线程中加任务了，有点问题
+    # with ThreadPoolExecutor(30) as thread_pool:
+    #     result = thread_pool.map(handle, range(2), [thread_pool] * 2)
+    #     print("start threading")
+    #     thread_pool.submit(handle, 111, thread_pool)
 
-[使用Python的 concurrent.futures 模块](https://python-parallel-programmning-cookbook.readthedocs.io/zh_CN/latest/chapter4/02_Using_the_concurrent.futures_Python_modules.html)
+```
 
-[python并发库：concurrent.futures的使用](https://blog.csdn.net/drdairen/article/details/69487643)
+参考链接
 
-[python并发 1：使用 futures 处理并发](https://segmentfault.com/a/1190000009819359)
+[使用Python进行并发编程-PoolExecutor篇](http://www.dongwm.com/archives/%E4%BD%BF%E7%94%A8Python%E8%BF%9B%E8%A1%8C%E5%B9%B6%E5%8F%91%E7%BC%96%E7%A8%8B-PoolExecutor%E7%AF%87/)  
+[使用Python的 concurrent.futures 模块](https://python-parallel-programmning-cookbook.readthedocs.io/zh_CN/latest/chapter4/02_Using_the_concurrent.futures_Python_modules.html)  
+[python并发库：concurrent.futures的使用](https://blog.csdn.net/drdairen/article/details/69487643)   
+[python并发 1：使用 futures 处理并发](https://segmentfault.com/a/1190000009819359)